@aj-archipelago/cortex 1.3.50 → 1.3.52

package/helper-apps/cortex-file-handler/index.js

@@ -1,440 +0,0 @@
-import { downloadFile, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, AZURE_STORAGE_CONTAINER_NAME, uploadChunkToGCS } from './blobHandler.js';
-import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
-import { ensureEncoded, ensureFileExtension, urlExists } from './helper.js';
-import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
-import { documentToText, easyChunker } from './docHelper.js';
-import { DOC_EXTENSIONS } from './constants.js';
-import path from 'path';
-import os from 'os';
-import { v4 as uuidv4 } from 'uuid';
-import fs from 'fs';
-
-const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
-const useGCS = process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY ? true : false;
-
-console.log(`Storage configuration - ${useAzure ? 'Azure' : 'Local'} Storage${useGCS ? ' and Google Cloud Storage' : ''}`);
-
-let isCleanupRunning = false;
-async function cleanupInactive(context) {
-    try {
-        if (isCleanupRunning) { return; } //no need to cleanup every call
-        isCleanupRunning = true;
-        const cleaned = await cleanupRedisFileStoreMap();
-
-        const cleanedAzure = [];
-        const cleanedLocal = [];
-        const cleanedGCS = [];
-
-        for(const key in cleaned){
-            const item = cleaned[key];
-            const {url,gcs} = item;
-            if(url){
-                if(url.includes('.blob.core.windows.net/')){ 
-                    cleanedAzure.push(url);
-                }else if(url.startsWith('gs://')){
-                    cleanedGCS.push(url);
-                }else{
-                    cleanedLocal.push(url);
-                }
-            }
-
-            if(item && item.gcs){
-                cleanedGCS.push(gcs);
-            }
-        }
-        
-        try {
-            if (cleanedAzure && cleanedAzure.length > 0) {
-                await cleanup(context, cleanedAzure);
-            }
-        } catch (error) {
-            console.log('Error occurred during azure cleanup:', error);
-        }
-
-        try {
-            if (cleanedLocal && cleanedLocal.length > 0) {
-                await cleanupLocal(cleanedLocal);
-            }
-        }catch(err){
-            console.log('Error occurred during local cleanup:', err);
-        }
-
-        try{
-            if(cleanedGCS && cleanedGCS.length > 0){
-                await cleanupGCS(cleanedGCS);
-            }
-        }catch(err){
-            console.log('Error occurred during GCS cleanup:', err);
-        }
-         
-    } catch (error) {
-        console.log('Error occurred during cleanup:', error);
-    } finally{
-        isCleanupRunning = false;
-    }
-}
-
-async function CortexFileHandler(context, req) {
-    const { uri, requestId, save, hash, checkHash, clearHash, fetch, load, restore } = req.body?.params || req.query;
-    const operation = save ? 'save' : 
-                     checkHash ? 'checkHash' : 
-                     clearHash ? 'clearHash' : 
-                     fetch || load || restore ? 'remoteFile' : 
-                     req.method.toLowerCase() === 'delete' || req.query.operation === 'delete' ? 'delete' :
-                     uri ? (DOC_EXTENSIONS.some(ext => uri.toLowerCase().endsWith(ext)) ? 'document_processing' : 'media_chunking') : 
-                     'upload';
-    
-    context.log(`Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ''}${uri ? `uri: ${uri}, ` : ''}${hash ? `hash: ${hash}, ` : ''}operation: ${operation}`);
-
-    cleanupInactive(context); //trigger & no need to wait for it
-
-    // Clean up blob when request delete which means processing marked completed
-    if (operation === 'delete') {
-        const deleteRequestId = req.query.requestId || requestId;
-        if (!deleteRequestId) {
-            context.res = {
-                status: 400,
-                body: "Please pass a requestId on the query string"
-            };
-            return;
-        }
-        
-        // Delete from Azure/Local storage
-        const azureResult = useAzure ? await deleteBlob(deleteRequestId) : await deleteFolder(deleteRequestId);
-        const gcsResult = [];
-        if (gcs) {
-            gcsResult.push(...await deleteGCS(deleteRequestId));
-        }
-        
-        context.res = {
-            status: 200,
-            body: { body: [...azureResult, ...gcsResult] }
-        };
-        return;
-    }
-
-    const remoteUrl = fetch || restore || load;
-    if (req.method.toLowerCase() === `get` && remoteUrl) {
-        context.log(`Remote file: ${remoteUrl}`);
-        let filename;  // Declare filename outside try block
-        try {
-            // Validate URL format and accessibility
-            const urlCheck = await urlExists(remoteUrl);
-            if (!urlCheck.valid) {
-                context.res = {
-                    status: 400,
-                    body: 'Invalid or inaccessible URL'
-                };
-                return;
-            }
-
-            // Check if file already exists (using hash as the key)
-            let exists = await getFileStoreMap(remoteUrl);
-            if(exists){
-                context.res = {
-                    status: 200,
-                    body: exists
-                };
-                //update redis timestamp with current time
-                await setFileStoreMap(remoteUrl, exists);
-                return;
-            }
-
-            // Download the file first
-            const urlObj = new URL(remoteUrl);
-            let originalFileName = path.basename(urlObj.pathname);
-            if (!originalFileName || originalFileName === '') {
-                originalFileName = urlObj.hostname;
-            }
-            
-            // Ensure the filename has the correct extension based on content type
-            originalFileName = ensureFileExtension(originalFileName, urlCheck.contentType);
-
-            const maxLength = 200; // Set the maximum length for the filename
-            let truncatedFileName = originalFileName;
-            if (originalFileName.length > maxLength) {
-                const extension = path.extname(originalFileName);
-                const basename = path.basename(originalFileName, extension);
-                truncatedFileName = basename.substring(0, maxLength - extension.length) + extension;
-            }
-
-            // Use the original-truncated file name when saving the downloaded file
-            filename = path.join(os.tmpdir(), truncatedFileName);
-            await downloadFile(remoteUrl, filename);
-            
-            // Now upload the downloaded file
-            const res = await uploadBlob(context, null, !useAzure, filename, remoteUrl);
-
-            //Update Redis (using hash as the key)
-            await setFileStoreMap(remoteUrl, res);
-
-            // Return the file URL
-            context.res = {
-                status: 200,
-                body: res,
-            };
-        } catch (error) {
-            context.log("Error processing remote file request:", error);
-            context.res = {
-                status: 500,
-                body: `Error processing file: ${error.message}`
-            };
-        } finally {
-            // Cleanup temp file if it exists
-            try {
-                if (filename && fs.existsSync(filename)) {
-                    fs.unlinkSync(filename);
-                }
-            } catch (err) {
-                context.log("Error cleaning up temp file:", err);
-            }
-        }
-        return;
-    }
-
-    if(hash && clearHash){
-        try {
-            const hashValue = await getFileStoreMap(hash);
-            if (hashValue) {
-                await removeFromFileStoreMap(hash);
-                context.res = {
-                    status: 200,
-                    body: `Hash ${hash} removed`
-                };
-            } else {
-                context.res = {
-                    status: 404,
-                    body: `Hash ${hash} not found`
-                };
-            }
-        } catch (error) {
-            context.res = {
-                status: 500,
-                body: `Error occurred during hash cleanup: ${error}`
-            };
-            console.log('Error occurred during hash cleanup:', error);
-        }
-        return;
-    }
-
-    if(hash && checkHash){ //check if hash exists
-        let hashResult = await getFileStoreMap(hash);
-
-        if(hashResult){
-            context.log(`File exists in map: ${hash}`);
-            
-            // Check primary storage (Azure/Local) first
-            const primaryExists = await urlExists(hashResult?.url);
-            const gcsExists = gcs ? await gcsUrlExists(hashResult?.gcs) : false;
-
-            // If neither storage has the file, remove from map and return not found
-            if (!primaryExists.valid && !gcsExists) {
-                context.log(`File not found in any storage. Removing from map: ${hash}`);
-                await removeFromFileStoreMap(hash);
-                context.res = {
-                    status: 404,
-                    body: `Hash ${hash} not found in storage`
-                };
-                return;
-            }
-
-            // If primary is missing but GCS exists, restore from GCS
-            if (!primaryExists.valid && gcsExists) {
-                context.log(`Primary storage file missing, restoring from GCS: ${hash}`);
-                try {
-                    const res = await CortexFileHandler(context, {
-                        method: 'GET',
-                        body: { params: { fetch: hashResult.gcs } }
-                    });
-                    if (res?.body?.url) {
-                        hashResult.url = res.body.url;
-                    }
-                } catch (error) {
-                    console.error('Error restoring from GCS:', error);
-                }
-            }
-            // If GCS is missing but primary exists, restore to GCS
-            else if (primaryExists.valid && gcs && !gcsExists) {
-                context.log(`GCS file missing, restoring from primary: ${hash}`);
-                const { gcs: _, ...fileInfo } = hashResult; // eslint-disable-line no-unused-vars
-                hashResult = await ensureGCSUpload(context, fileInfo);
-            }
-
-            // Final check to ensure we have at least one valid storage location
-            const finalPrimaryCheck = await urlExists(hashResult?.url);
-            if (!finalPrimaryCheck.valid && !await gcsUrlExists(hashResult?.gcs)) {
-                context.log(`Failed to restore file. Removing from map: ${hash}`);
-                await removeFromFileStoreMap(hash);
-                context.res = {
-                    status: 404,
-                    body: `Hash ${hash} not found and restoration failed`
-                };
-                return;
-            }
-
-            //update redis timestamp with current time
-            await setFileStoreMap(hash, hashResult);
-
-            context.res = {
-                status: 200,
-                body: hashResult
-            };
-            return;
-        }
-
-        context.res = {
-            status: 404,
-            body: `Hash ${hash} not found`
-        };
-        return;
-    }
-
-    if (req.method.toLowerCase() === `post`) {
-        await uploadBlob(context, req, !useAzure, null, hash);
-        if(hash && context?.res?.body){ 
-            await setFileStoreMap(hash, context.res.body);
-        }
-        return
-    }
-
-    if (!uri || !requestId) {
-        context.res = {
-            status: 400,
-            body: "Please pass a uri and requestId on the query string or in the request body"
-        };
-        return;
-    }
-
-    let totalCount = 0;
-    let completedCount = 0;
-    let numberOfChunks;
-
-    let file = ensureEncoded(uri); // encode url to handle special characters
-
-    const result = [];
-
-    const sendProgress = async (data = null) => {
-        completedCount++;
-        const progress = completedCount / totalCount;
-        await publishRequestProgress({ requestId, progress, completedCount, totalCount, numberOfChunks, data });
-    }
-
-    try {
-        // Parse URL and get pathname without query parameters for extension check
-        const urlObj = new URL(uri);
-        const pathWithoutQuery = urlObj.pathname;
-        
-        if (DOC_EXTENSIONS.some(ext => pathWithoutQuery.toLowerCase().endsWith(ext))) {
-            const extension = path.extname(pathWithoutQuery).toLowerCase();
-            const tempDir = path.join(os.tmpdir(), `${uuidv4()}`);
-            fs.mkdirSync(tempDir);
-            const downloadedFile = path.join(tempDir, `${uuidv4()}${extension}`);
-            await downloadFile(uri, downloadedFile);
-            const text = await documentToText(downloadedFile);
-            let tmpPath;
-
-            try {
-                if (save) {
-                    const fileName = `${uuidv4()}.txt`; // generate unique file name
-                    const filePath = path.join(tempDir, fileName);
-                    tmpPath = filePath;
-                    fs.writeFileSync(filePath, text); // write text to file
-            
-                    // save file to the cloud or local file system
-                    const saveResult = useAzure ? await saveFileToBlob(filePath, requestId) : await moveFileToPublicFolder(filePath, requestId);
-                    result.push(saveResult);
-
-                } else {
-                    result.push(...easyChunker(text));
-                }
-            } catch(err) {
-                console.log(`Error saving file ${uri} with request id ${requestId}:`, err);
-            } finally {
-                try {
-                    // delete temporary files
-                    tmpPath && fs.unlinkSync(tmpPath);
-                    downloadedFile && fs.unlinkSync(downloadedFile);
-                    console.log(`Cleaned temp files ${tmpPath}, ${downloadedFile}`);
-                } catch(err) {
-                    console.log(`Error cleaning temp files ${tmpPath}, ${downloadedFile}:`, err);
-                }
-                
-                try {
-                    //delete uploaded prev nontext file
-                    //check cleanup for uploaded files url
-                    const regex = new RegExp(`${AZURE_STORAGE_CONTAINER_NAME}/([a-z0-9-]+)`);
-                    const match = uri.match(regex);
-                    if (match && match[1]) {
-                        const extractedValue = match[1];
-                        useAzure ? await deleteBlob(extractedValue) : await deleteFolder(extractedValue);
-                        console.log(`Cleaned temp file ${uri} with request id ${extractedValue}`);
-                    }
-                } catch(err) {
-                    console.log(`Error cleaning temp file ${uri}:`, err);
-                }
-            }
-        } else {
-            const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(file);
-
-            numberOfChunks = chunkPromises.length; // for progress reporting
-            totalCount += chunkPromises.length * 4; // 4 steps for each chunk (download and upload)
-
-            // sequential download of chunks
-            const chunks = [];
-            for (const chunkPromise of chunkPromises) {
-                const chunkPath = await chunkPromise;
-                chunks.push(chunkPath);
-                await sendProgress();
-            }
-
-            // sequential processing of chunks
-            for (let index = 0; index < chunks.length; index++) {
-                const chunkPath = chunks[index];
-                let blobName;
-                let gcsUrl;
-                
-                if (useAzure) {
-                    blobName = await saveFileToBlob(chunkPath, requestId);
-                } else {
-                    blobName = await moveFileToPublicFolder(chunkPath, requestId);
-                }
-                
-                // If GCS is configured, save to GCS
-                gcsUrl = await uploadChunkToGCS(chunkPath, requestId);
-                
-                const chunkOffset = chunkOffsets[index];
-                result.push({ uri: blobName, offset: chunkOffset, gcs: gcsUrl });
-                console.log(`Saved chunk as: ${blobName}${gcsUrl ? ` and ${gcsUrl}` : ''}`);
-                await sendProgress();
-            }
-
-            // Cleanup the temp directory
-            try {
-                if (uniqueOutputPath && fs.existsSync(uniqueOutputPath)) {
-                    fs.rmSync(uniqueOutputPath, { recursive: true });
-                    console.log(`Cleaned temp directory: ${uniqueOutputPath}`);
-                }
-            } catch (err) {
-                console.log(`Error cleaning temp directory ${uniqueOutputPath}:`, err);
-            }
-        }
-    } catch (error) {
-        console.error("An error occurred:", error);
-        context.res = {
-            status: 500,
-            body: error.message || error
-        };
-        return;
-    }
-
-    console.log('result:', result.map(item =>
-        typeof item === 'object' ? JSON.stringify(item, null, 2) : item
-    ).join('\n'));
-    
-    context.res = {
-        body: result
-    };
-}
-
-export default CortexFileHandler;

package/helper-apps/cortex-file-handler/localFileHandler.js

@@ -1,108 +0,0 @@
-import { promises as fs } from 'fs';
-import { join, basename } from 'path';
-import { v4 as uuidv4 } from 'uuid';
-
-import { publicFolder, port, ipAddress } from "./start.js";
-
-async function moveFileToPublicFolder(chunkPath, requestId) {
-    // Use the filename with a UUID as the blob name
-    const filename = `${requestId}/${uuidv4()}_${basename(chunkPath)}`;
-
-    // Create the target folder if it doesn't exist
-    const targetFolder = join(publicFolder, requestId);
-    await fs.mkdir(targetFolder, { recursive: true });
-
-    // Move the file to the target folder
-    const targetPath = join(targetFolder, basename(filename));
-    await fs.rename(chunkPath, targetPath);
-
-    // Return the complete URL of the file
-    const fileUrl = `http://${ipAddress}:${port}/files/${filename}`;
-    // const fileUrl = `http://localhost:${port}/files/${filename}`;
-    return fileUrl;
-}
-
-async function deleteFolder(requestId) {
-    if (!requestId) throw new Error('Missing requestId parameter');
-    const targetFolder = join(publicFolder, requestId);
-    try {
-        // Check if folder exists first
-        const stats = await fs.stat(targetFolder);
-        if (stats.isDirectory()) {
-            // Get list of files before deleting
-            const files = await fs.readdir(targetFolder);
-            const deletedFiles = files.map(file => join(requestId, file));
-            // Delete the folder
-            await fs.rm(targetFolder, { recursive: true });
-            console.log(`Cleaned folder: ${targetFolder}`);
-            return deletedFiles;
-        }
-        return [];
-    } catch (error) {
-        if (error.code === 'ENOENT') {
-            // Folder doesn't exist, return empty array
-            return [];
-        }
-        throw error;
-    }
-}
-
-async function cleanupLocal(urls=null) {
-  const cleanedUrls = [];
-  if(!urls){
-    try {
-      // Read the directory
-      const items = await fs.readdir(publicFolder);
-
-      // Calculate the date that is x months ago
-      const monthsAgo = new Date();
-      monthsAgo.setMonth(monthsAgo.getMonth() - 1);
-
-      // Iterate through the items
-      for (const item of items) {
-        const itemPath = join(publicFolder, item);
-
-        // Get the stats of the item
-        const stats = await fs.stat(itemPath);
-
-        // Check if the item is a file or a directory
-        const isDirectory = stats.isDirectory();
-
-        // Compare the last modified date with three months ago
-        if (stats.mtime < monthsAgo) {
-          if (isDirectory) {
-            // If it's a directory, delete it recursively
-            await fs.rm(itemPath, { recursive: true });
-            console.log(`Cleaned directory: ${item}`);
-          } else {
-            // If it's a file, delete it
-            await fs.unlink(itemPath);
-            console.log(`Cleaned file: ${item}`);
-
-            // Add the URL of the cleaned file to cleanedUrls array
-            cleanedUrls.push(`http://${ipAddress}:${port}/files/${item}`);
-          }
-        }
-      }
-    } catch (error) {
-      console.error(`Error cleaning up files: ${error}`);
-    }
-  }else{
-    try{
-      for (const url of urls) {
-        const filename = url.split('/').pop();
-        const itemPath = join(publicFolder, filename);
-        await fs.unlink(itemPath);
-      }
-    }catch(error){
-      console.error(`Error cleaning up files: ${error}`);
-    }
-  }
-
-  // Return the array of cleaned file URLs
-  return cleanedUrls;
-}
-
-export {
-    moveFileToPublicFolder, deleteFolder, cleanupLocal
-};

package/helper-apps/cortex-file-handler/start.js

@@ -1,63 +0,0 @@
-import CortexFileHandler from "./index.js";
-import express from "express";
-import { fileURLToPath } from 'url';
-import { dirname, join } from 'path';
-import cors from 'cors';
-import { readFileSync } from 'fs';
-
-import { publicIpv4 } from 'public-ip';
-const ipAddress = await publicIpv4();
-
-const app = express();
-const port = process.env.PORT || 7071;
-const publicFolder = join(dirname(fileURLToPath(import.meta.url)), 'files');
-
-// Get version from package.json
-const packageJson = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'package.json'), 'utf8'));
-const version = packageJson.version;
-
-app.use(cors());
-// Serve static files from the public folder
-app.use('/files', express.static(publicFolder));
-
-// Health check endpoint
-app.get('/health', (req, res) => {
-    res.status(200).json({
-        status: 'healthy',
-        version: version
-    });
-});
-
-// New primary endpoint
-app.all('/api/CortexFileHandler', async (req, res) => {
-    const context = { req, res, log: console.log }
-    try {
-        await CortexFileHandler(context, req);
-        context.log(context.res);
-        res.status(context.res.status || 200).send(context.res.body);
-    } catch (error) {
-        const status = error.status || 500;
-        const message = error.message || 'Internal server error';
-        res.status(status).send(message);
-    }
-});
-
-// Legacy endpoint for compatibility
-app.all('/api/MediaFileChunker', async (req, res) => {
-    const context = { req, res, log: console.log }
-    try {
-        await CortexFileHandler(context, req);
-        context.log(context.res);
-        res.status(context.res.status || 200).send(context.res.body);
-    } catch (error) {
-        const status = error.status || 500;
-        const message = error.message || 'Internal server error';
-        res.status(status).send(message);
-    }
-});
-
-app.listen(port, () => {
-    console.log(`Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`);
-});
-
-export { port, publicFolder, ipAddress };