@aj-archipelago/cortex 1.3.10 → 1.3.12

package/helper-apps/cortex-file-handler/constants.js

@@ -0,0 +1,107 @@
+export const DOC_EXTENSIONS = [
+    ".txt",
+    ".json",
+    ".csv",
+    ".md",
+    ".xml",
+    ".js",
+    ".html",
+    ".css",
+    ".doc",
+    ".docx",
+    ".xls",
+    ".xlsx"
+];
+
+export const IMAGE_EXTENSIONS = [
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".webp",
+    ".heic",
+    ".heif",
+    ".pdf"
+];
+  
+export const VIDEO_EXTENSIONS = [
+    ".mp4",
+    ".mpeg",
+    ".mov",
+    ".avi",
+    ".flv",
+    ".mpg",
+    ".webm",
+    ".wmv",
+    ".3gp"
+];
+  
+export const AUDIO_EXTENSIONS = [
+    ".wav",
+    ".mp3",
+    ".aac",
+    ".ogg",
+    ".flac"
+];
+
+export const ACCEPTED_MIME_TYPES = {
+    // Document types
+    'text/plain': ['.txt'],
+    'application/json': ['.json'],
+    'text/csv': ['.csv'],
+    'text/markdown': ['.md'],
+    'application/xml': ['.xml'],
+    'text/javascript': ['.js'],
+    'text/html': ['.html'],
+    'text/css': ['.css'],
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'],
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
+    'application/msword': ['.doc'],
+    'application/vnd.ms-excel': ['.xls'],
+    'application/vnd.ms-word.document.macroEnabled.12': ['.docm'],
+    'application/vnd.ms-excel.sheet.macroEnabled.12': ['.xlsm'],
+    'application/vnd.ms-word.template.macroEnabled.12': ['.dotm'],
+    'application/vnd.ms-excel.template.macroEnabled.12': ['.xltm'],
+    
+    // Image types
+    'image/jpeg': ['.jpg', '.jpeg'],
+    'image/png': ['.png'],
+    'image/webp': ['.webp'],
+    'image/heic': ['.heic'],
+    'image/heif': ['.heif'],
+    'application/pdf': ['.pdf'],
+    
+    // Audio types
+    'audio/wav': ['.wav'],
+    'audio/mpeg': ['.mp3'],
+    'audio/aac': ['.aac'],
+    'audio/ogg': ['.ogg'],
+    'audio/flac': ['.flac'],
+    
+    // Video types
+    'video/mp4': ['.mp4'],
+    'video/mpeg': ['.mpeg', '.mpg'],
+    'video/quicktime': ['.mov'],
+    'video/x-msvideo': ['.avi'],
+    'video/x-flv': ['.flv'],
+    'video/webm': ['.webm'],
+    'video/x-ms-wmv': ['.wmv'],
+    'video/3gpp': ['.3gp']
+};
+
+// Helper function to check if a mime type is accepted
+export function isAcceptedMimeType(mimeType) {
+    return mimeType in ACCEPTED_MIME_TYPES;
+}
+
+// Helper function to get accepted extensions for a mime type
+export function getExtensionsForMimeType(mimeType) {
+    return ACCEPTED_MIME_TYPES[mimeType] || [];
+}
+
+// Helper function to check if an extension is accepted
+export function isAcceptedExtension(extension) {
+    return DOC_EXTENSIONS.includes(extension) || 
+           IMAGE_EXTENSIONS.includes(extension) || 
+           VIDEO_EXTENSIONS.includes(extension) || 
+           AUDIO_EXTENSIONS.includes(extension);
+} 

package/helper-apps/cortex-file-handler/docHelper.js

@@ -1,4 +1,4 @@
-import pdfjsLib from 'pdfjs-dist';
+import * as pdfjsLib from 'pdfjs-dist';
 import fs from 'fs/promises';
 import mammoth from 'mammoth';
 import XLSX from 'xlsx';
@@ -95,10 +95,13 @@ export async function documentToText(filePath) {
         case 'pdf':
             return pdfToText(filePath);
         case 'txt':
+        case 'html':
             return txtToText(filePath);
         case 'docx':
+        case 'doc':
             return docxToText(filePath);
         case 'xlsx':
+        case 'xls':
             return xlsxToText(filePath);
         case 'csv':
             return csvToText(filePath);

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -6,170 +6,232 @@ import os from 'os';
 import { promisify } from 'util';
 import axios from 'axios';
 import { ensureEncoded } from './helper.js';
-import ytdl from '@distube/ytdl-core';
-
+import http from 'http';
+import https from 'https';
+import { pipeline } from 'stream/promises';
 
 const ffmpegProbe = promisify(ffmpeg.ffprobe);
 
+// Temp file management
+const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+const tempDirectories = new Map(); // dir -> { createdAt, requestId }
+
+// Temp directory cleanup
+async function cleanupTempDirectories() {
+    const tempDir = os.tmpdir();
+    
+    for (const [dir, info] of tempDirectories) {
+        try {
+            // Cleanup directories older than 1 hour
+            if (Date.now() - info.createdAt > 60 * 60 * 1000) {
+                await fs.promises.rm(dir, { recursive: true, force: true });
+                tempDirectories.delete(dir);
+                console.log(`Cleaned up old temp directory: ${dir}`);
+            }
+        } catch (err) {
+            // Directory might be gone
+            tempDirectories.delete(dir);
+        }
+    }
+}
+
+// Setup periodic cleanup
+setInterval(async () => {
+    try {
+        await cleanupTempDirectories();
+    } catch (err) {
+        console.error('Error during periodic cleanup:', err);
+    }
+}, CLEANUP_INTERVAL_MS);
 
+// Process a single chunk with streaming
 async function processChunk(inputPath, outputFileName, start, duration) {
     return new Promise((resolve, reject) => {
-        ffmpeg(inputPath)
+        const command = ffmpeg(inputPath)
             .seekInput(start)
             .duration(duration)
             .format('mp3')
             .audioCodec('libmp3lame')
             .audioBitrate(128)
-            .on('start', (cmd) => {
-                console.log(`Started FFmpeg with command: ${cmd}`);
+            .on('start', () => {
+                console.log(`Processing chunk: ${start}s -> ${start + duration}s`);
             })
-            .on('error', (err) => {
-                console.error(`Error occurred while processing chunk:`, err);
+            .on('progress', (progress) => {
+                if (progress.percent) {
+                    console.log(`Chunk progress: ${progress.percent}%`);
+                }
+            })
+            .on('error', (err, stdout, stderr) => {
+                console.error('FFmpeg error:', err.message);
+                if (stdout) console.log('FFmpeg stdout:', stdout);
+                if (stderr) console.error('FFmpeg stderr:', stderr);
                 reject(err);
             })
             .on('end', () => {
-                console.log(`Finished processing chunk`);
+                console.log(`Chunk complete: ${outputFileName}`);
                 resolve(outputFileName);
-            })
-            .save(outputFileName);
+            });
+
+        // Use pipe() to handle streaming
+        command.pipe(fs.createWriteStream(outputFileName), { end: true });
     });
 }
 
 const generateUniqueFolderName = () => {
     const uniqueFolderName = uuidv4();
-    const tempFolderPath = os.tmpdir(); // Get the system's temporary folder
-    const uniqueOutputPath = path.join(tempFolderPath, uniqueFolderName);
-    return uniqueOutputPath;
+    const tempFolderPath = os.tmpdir();
+    return path.join(tempFolderPath, uniqueFolderName);
 }
 
 async function downloadFile(url, outputPath) {
     try {
         let response;
         try {
-            response = await axios.get(decodeURIComponent(url), { responseType: 'stream' });
+            response = await axios.get(decodeURIComponent(url), { 
+                responseType: 'stream',
+                // Add timeout and maxContentLength
+                timeout: 30000,
+                maxContentLength: Infinity,
+                // Enable streaming download
+                decompress: true,
+                // Use a smaller chunk size for better memory usage
+                httpAgent: new http.Agent({ keepAlive: true }),
+                httpsAgent: new https.Agent({ keepAlive: true })
+            });
         } catch (error) {
-            response = await axios.get(url, { responseType: 'stream' });
+            response = await axios.get(url, { 
+                responseType: 'stream',
+                timeout: 30000,
+                maxContentLength: Infinity,
+                decompress: true,
+                httpAgent: new http.Agent({ keepAlive: true }),
+                httpsAgent: new https.Agent({ keepAlive: true })
+            });
         }
 
-        // Make an HTTP request for the file
-
-        // Create a writable file stream to save the file
-        const fileStream = fs.createWriteStream(outputPath);
-
-        // Pipe the response data into the file stream
-        response.data.pipe(fileStream);
-
-        // Wait for the file stream to finish writing
-        await new Promise((resolve, reject) => {
-            fileStream.on('finish', resolve);
-            fileStream.on('error', reject);
-        });
+        const writer = fs.createWriteStream(outputPath);
+        
+        // Use pipeline for better error handling and memory management
+        await pipeline(response.data, writer);
 
-        console.log(`Downloaded file saved to: ${outputPath}`);
+        if (!fs.existsSync(outputPath) || fs.statSync(outputPath).size === 0) {
+            throw new Error('Download failed or file is empty');
+        }
     } catch (error) {
-        console.error(`Error downloading file from ${url}:`, error);
+        if (fs.existsSync(outputPath)) {
+            fs.unlinkSync(outputPath);
+        }
         throw error;
     }
 }
 
-// Split a media file into chunks of max 500 seconds
-async function splitMediaFile(inputPath, chunkDurationInSeconds = 500) {
+async function splitMediaFile(inputPath, chunkDurationInSeconds = 500, requestId = uuidv4()) {
+    let tempPath = null;
+    let uniqueOutputPath = null;
+    let inputStream = null;
+    
     try {
-        // Create unique folder
-        const uniqueOutputPath = generateUniqueFolderName();
+        uniqueOutputPath = generateUniqueFolderName();
         fs.mkdirSync(uniqueOutputPath, { recursive: true });
+        
+        tempDirectories.set(uniqueOutputPath, {
+            createdAt: Date.now(),
+            requestId
+        });
 
-        // Download the file if it's not a local file
+        // Handle URL downloads with streaming
         const isUrl = /^(https?|ftp):\/\/[^\s/$.?#].[^\s]*$/i.test(inputPath);
         if (isUrl) {
-            inputPath = ensureEncoded(inputPath);
-            // Extract the original file name from the URL
-            const urlObj = new URL(inputPath);
-            const originalFileName = path.basename(urlObj.pathname);
-            const maxLength = 200; // Set the maximum length for the filename
-            let truncatedFileName = originalFileName;
-            if (originalFileName.length > maxLength) {
-                const extension = path.extname(originalFileName); // Preserve the file extension
-                const basename = path.basename(originalFileName, extension); // Get the filename without the extension
-                truncatedFileName = basename.substring(0, maxLength) + extension; // Truncate the filename and append the extension
-            }
+            const urlObj = new URL(ensureEncoded(inputPath));
+            const originalFileName = path.basename(urlObj.pathname) || 'downloaded_file';
+            tempPath = path.join(uniqueOutputPath, originalFileName);
+            console.log('Downloading file to:', tempPath);
+            await downloadFile(inputPath, tempPath);
+            inputPath = tempPath;
+        }
 
-            // Use the original-truncated file name when saving the downloaded file
-            const downloadPath = path.join(uniqueOutputPath, truncatedFileName);
-            await downloadFile(inputPath, downloadPath);
-            inputPath = downloadPath;
+        inputPath = path.resolve(inputPath);
+        if (!fs.existsSync(inputPath)) {
+            throw new Error(`Input file not found: ${inputPath}`);
         }
 
+        // Use a larger chunk size for better throughput while still managing memory
+        inputStream = fs.createReadStream(inputPath, { 
+            highWaterMark: 4 * 1024 * 1024, // 4MB chunks
+            autoClose: true
+        });
+
+        console.log('Probing file:', inputPath);
         const metadata = await ffmpegProbe(inputPath);
+        if (!metadata?.format?.duration) {
+            throw new Error('Invalid media file or unable to determine duration');
+        }
+
         const duration = metadata.format.duration;
         const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
-
-        const chunkPromises = [];
-        const chunkOffsets = [];
-
-        for (let i = 0; i < numChunks; i++) {
-            const outputFileName = path.join(uniqueOutputPath, `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`);
-            const offset = i * chunkDurationInSeconds;
-
-            const chunkPromise = processChunk(inputPath, outputFileName, offset, chunkDurationInSeconds);
+        console.log(`Processing ${numChunks} chunks of ${chunkDurationInSeconds} seconds each`);
+
+        const chunkResults = new Array(numChunks); // Pre-allocate array to maintain order
+        const chunkOffsets = new Array(numChunks); // Pre-allocate offsets array
+
+        // Process chunks in parallel with a concurrency limit
+        const CONCURRENT_CHUNKS = 3; // Process 3 chunks at a time
+        for (let i = 0; i < numChunks; i += CONCURRENT_CHUNKS) {
+            const chunkBatch = [];
+            for (let j = 0; j < CONCURRENT_CHUNKS && i + j < numChunks; j++) {
+                const chunkIndex = i + j;
+                const outputFileName = path.join(uniqueOutputPath, `chunk-${chunkIndex + 1}-${path.parse(inputPath).name}.mp3`);
+                const offset = chunkIndex * chunkDurationInSeconds;
+                
+                chunkBatch.push(processChunk(inputPath, outputFileName, offset, chunkDurationInSeconds)
+                    .then(result => {
+                        chunkResults[chunkIndex] = result; // Store in correct position
+                        chunkOffsets[chunkIndex] = offset; // Store offset in correct position
+                        console.log(`Completed chunk ${chunkIndex + 1}/${numChunks}`);
+                        return result;
+                    })
+                    .catch(error => {
+                        console.error(`Failed to process chunk ${chunkIndex + 1}:`, error);
+                        return null;
+                    }));
+            }
             
-            chunkPromises.push(chunkPromise);
-            chunkOffsets.push(offset);
+            // Wait for the current batch to complete before starting the next
+            await Promise.all(chunkBatch);
         }
 
-        return { chunkPromises, chunkOffsets, uniqueOutputPath }; 
-    } catch (err) {
-        const msg = `Error processing media file, check if the file is a valid media file or is accessible`;
-        console.error(msg, err);
-        throw new Error(msg);
-    }
-}
-
-const ytdlDownload = async (url, filename, video = false) => {
-    return new Promise((resolve, reject) => {
-        const videoOptions = video 
-            ? { filter: 'audioandvideo' }  // audio and video
-            : { quality: 'highestaudio' }; // audio only
-
-        const encodedUrl = encodeURI(url);
-        const videoStream = ytdl(encodedUrl, videoOptions);
-        let lastLoggedTime = Date.now();
+        // Filter out any failed chunks
+        const validChunks = chunkResults.filter(Boolean);
+        const validOffsets = chunkOffsets.filter((_, index) => chunkResults[index]);
 
-        videoStream.on('error', (error) => {
-            reject(error);
-        });
+        if (validChunks.length === 0) {
+            throw new Error('No chunks were successfully processed');
+        }
 
-        videoStream.on('progress', (chunkLength, downloaded, total) => {
-            const currentTime = Date.now();
-            if (currentTime - lastLoggedTime >= 2000) { // Log every 2 seconds
-                const percent = downloaded / total;
-                console.log(`${(percent * 100).toFixed(2)}% downloaded ${url}`);
-                lastLoggedTime = currentTime;
+        return { chunkPromises: validChunks, chunkOffsets: validOffsets, uniqueOutputPath };
+    } catch (err) {
+        if (uniqueOutputPath && fs.existsSync(uniqueOutputPath)) {
+            try {
+                fs.rmSync(uniqueOutputPath, { recursive: true, force: true });
+                tempDirectories.delete(uniqueOutputPath);
+            } catch (cleanupErr) {
+                console.error('Error during cleanup:', cleanupErr);
             }
-        });
-
-        videoStream.pipe(fs.createWriteStream(filename))
-            .on('finish', () => {
-                resolve();
-            })
-            .on('error', (error) => {
-                reject(error);
-            });
-    });
-};
-
-async function processYoutubeUrl(url, video=false) {
-    try {
-        const outputFormat = video ? '.mp4' : '.mp3';
-        const outputFileName = path.join(os.tmpdir(), `${uuidv4()}${outputFormat}`);
-        await ytdlDownload(url, outputFileName, video);
-        return outputFileName;
-    } catch (e) {
-        console.log(e);
-        throw new Error(`Error processing YouTube video, YouTube downloader might be outdated or blocked. ${e.message}`);    
+        }
+        console.error('Error in splitMediaFile:', err);
+        throw new Error(`Error processing media file: ${err.message}`);
+    } finally {
+        if (inputStream) {
+            try {
+                inputStream.destroy();
+            } catch (err) {
+                console.error('Error closing input stream:', err);
+            }
+        }
     }
 }
 
 export {
-    splitMediaFile, processYoutubeUrl, downloadFile
+    splitMediaFile,
+    downloadFile
 };

package/helper-apps/cortex-file-handler/helper.js

@@ -1,11 +1,8 @@
 import fs from 'fs';
+import { ACCEPTED_MIME_TYPES } from './constants.js';
+import path from 'path';
 
-function isValidYoutubeUrl(url) {
-    const regex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.?be)\/.+$/;
-    return regex.test(url);
-}
-
-async function deleteTempPath(path) {
+export async function deleteTempPath(path) {
     try {
         if (!path) {
             console.log('Temporary path is not defined.');
@@ -28,18 +25,43 @@ async function deleteTempPath(path) {
     }
 }
 
-function ensureEncoded(url) {
+// Get the first extension for a given mime type
+export function getExtensionForMimeType(mimeType) {
+    if (!mimeType) return '';
+    const cleanMimeType = mimeType.split(';')[0].trim();
+    const extensions = ACCEPTED_MIME_TYPES[cleanMimeType];
+    return extensions ? extensions[0] : '';
+}
+
+// Ensure a filename has the correct extension based on its mime type
+export function ensureFileExtension(filename, mimeType) {
+    if (!mimeType) return filename;
+    
+    const extension = getExtensionForMimeType(mimeType);
+    if (!extension) return filename;
+
+    // If filename already has this extension, return as is
+    if (filename.toLowerCase().endsWith(extension)) {
+        return filename;
+    }
+
+    // Get the current extension if any
+    const currentExt = path.extname(filename);
+    
+    // If there's no current extension, just append the new one
+    if (!currentExt) {
+        return `${filename}${extension}`;
+    }
+    
+    // Replace the current extension with the new one
+    return filename.slice(0, -currentExt.length) + extension;
+}
+
+export function ensureEncoded(url) {
     try {
-        const decodedUrl = decodeURI(url);
-        if (decodedUrl === url) {
-            return encodeURI(url);
-        }
-        return url;
-    } catch (e) {
+        return decodeURIComponent(url) !== url ? url : encodeURI(url);
+    } catch (error) {
+        console.error('Error encoding URL:', error);
         return url;
     }
 }
-
-export {
-    isValidYoutubeUrl, deleteTempPath, ensureEncoded
-}