@aj-archipelago/cortex 1.3.49 → 1.3.51

package/helper-apps/cortex-file-handler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "1.0.17",
+  "version": "1.1.01",
   "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "scripts": {
@@ -21,7 +21,7 @@
     "express": "^4.21.1",
     "fluent-ffmpeg": "^2.1.3",
     "ioredis": "^5.3.1",
-    "mammoth": "^1.6.0",
+    "mime-types": "^3.0.1",
     "papaparse": "^5.4.1",
     "pdfjs-dist": "^4.2.67",
     "public-ip": "^6.0.1",
@@ -29,13 +29,17 @@
     "xlsx": "^0.18.5"
   },
   "devDependencies": {
+    "@eslint/js": "^9.26.0",
     "ava": "^5.3.1",
     "dotenv": "^16.3.1",
-    "nock": "^13.3.0"
+    "eslint-plugin-import": "^2.31.0",
+    "globals": "^16.1.0",
+    "nock": "^13.3.0",
+    "typescript-eslint": "^8.32.1"
   },
   "ava": {
     "files": [
-      "tests/**/*",
+      "tests/**/*.test.js",
       "!tests/test-files/**/*",
       "!tests/test-docs/**/*",
       "!tests/mocks/**/*"

package/helper-apps/cortex-file-handler/redis.js

@@ -1,5 +1,5 @@
 import redis from 'ioredis';
-const connectionString = process.env["REDIS_CONNECTION_STRING"];
+const connectionString = process.env['REDIS_CONNECTION_STRING'];
 const client = redis.createClient(connectionString);
 // client.connect();
 
@@ -30,7 +30,7 @@ const publishRequestProgress = async (data) => {
 // Function to get all key value pairs in "FileStoreMap" hash map
 const getAllFileStoreMap = async () => {
     try {
-        const allKeyValuePairs = await client.hgetall("FileStoreMap");
+        const allKeyValuePairs = await client.hgetall('FileStoreMap');
         // Parse each JSON value in the returned object
         for (const key in allKeyValuePairs) {
             try {
@@ -43,7 +43,9 @@ const getAllFileStoreMap = async () => {
         }
         return allKeyValuePairs;
     } catch (error) {
-        console.error(`Error getting all key-value pairs from FileStoreMap: ${error}`);
+        console.error(
+            `Error getting all key-value pairs from FileStoreMap: ${error}`,
+        );
         return {}; // Return null or any default value indicating an error occurred
     }
 };
@@ -52,7 +54,7 @@ const getAllFileStoreMap = async () => {
 const setFileStoreMap = async (key, value) => {
     try {
         value.timestamp = new Date().toISOString();
-        await client.hset("FileStoreMap", key, JSON.stringify(value));
+        await client.hset('FileStoreMap', key, JSON.stringify(value));
     } catch (error) {
         console.error(`Error setting key in FileStoreMap: ${error}`);
     }
@@ -60,7 +62,7 @@ const setFileStoreMap = async (key, value) => {
 
 const getFileStoreMap = async (key) => {
     try {
-        const value = await client.hget("FileStoreMap", key);
+        const value = await client.hget('FileStoreMap', key);
         if (value) {
             try {
                 // parse the value back to an object before returning
@@ -80,9 +82,9 @@ const getFileStoreMap = async (key) => {
 // Function to remove key from "FileStoreMap" hash map
 const removeFromFileStoreMap = async (key) => {
     try {
-        // hdel returns the number of keys that were removed.
-        // If the key does not exist, 0 is returned.
-        const result = await client.hdel("FileStoreMap", key);
+    // hdel returns the number of keys that were removed.
+    // If the key does not exist, 0 is returned.
+        const result = await client.hdel('FileStoreMap', key);
         if (result === 0) {
             console.log(`The key ${key} does not exist`);
         } else {
@@ -93,31 +95,35 @@ const removeFromFileStoreMap = async (key) => {
     }
 };
 
-const cleanupRedisFileStoreMap = async (nDays=1) => {
-    let cleaned = [];
+const cleanupRedisFileStoreMap = async (nDays = 1) => {
+    const cleaned = [];
     try {
         const map = await getAllFileStoreMap();
         const nDaysAgo = new Date(Date.now() - nDays * 24 * 60 * 60 * 1000);
 
-        for(const key in map){
+        for (const key in map) {
             const value = map[key];
             const timestamp = value?.timestamp ? new Date(value.timestamp) : null;
-            if(!timestamp || timestamp.getTime() < nDaysAgo.getTime()){
+            if (!timestamp || timestamp.getTime() < nDaysAgo.getTime()) {
                 // Remove the key from the "FileStoreMap" hash map
                 await removeFromFileStoreMap(key);
                 console.log(`Removed key ${key} from FileStoreMap`);
-                cleaned.push(Object.assign({hash:key}, value));
+                cleaned.push(Object.assign({ hash: key }, value));
             }
         }
     } catch (error) {
         console.error(`Error cleaning FileStoreMap: ${error}`);
     } finally {
-        // Cleanup code if needed
+    // Cleanup code if needed
     }
     return cleaned;
 };
 
-
 export {
-    publishRequestProgress, connectClient, setFileStoreMap, getFileStoreMap, removeFromFileStoreMap, cleanupRedisFileStoreMap
-};
+    publishRequestProgress,
+    connectClient,
+    setFileStoreMap,
+    getFileStoreMap,
+    removeFromFileStoreMap,
+    cleanupRedisFileStoreMap,
+};

package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js

@@ -2,21 +2,24 @@ import { BlobServiceClient } from '@azure/storage-blob';
 
 async function createContainer() {
     try {
-        const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
-        const containerClient = blobServiceClient.getContainerClient("test-container");
-        
-        console.log("Creating container...");
+        const blobServiceClient = BlobServiceClient.fromConnectionString(
+            'UseDevelopmentStorage=true',
+        );
+        const containerClient =
+      blobServiceClient.getContainerClient('test-container');
+
+        console.log('Creating container...');
         await containerClient.create();
-        console.log("Container created successfully");
+        console.log('Container created successfully');
     } catch (error) {
-        // Ignore if container already exists
+    // Ignore if container already exists
         if (error.statusCode === 409) {
-            console.log("Container already exists");
+            console.log('Container already exists');
         } else {
-            console.error("Error creating container:", error);
+            console.error('Error creating container:', error);
             process.exit(1);
         }
     }
 }
 
-createContainer(); 
+createContainer();

package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js

@@ -3,18 +3,21 @@ import { Storage } from '@google-cloud/storage';
 
 async function createAzureContainer() {
     try {
-        const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
-        const containerClient = blobServiceClient.getContainerClient("test-container");
-        
-        console.log("Creating Azure container...");
+        const blobServiceClient = BlobServiceClient.fromConnectionString(
+            'UseDevelopmentStorage=true',
+        );
+        const containerClient =
+      blobServiceClient.getContainerClient('test-container');
+
+        console.log('Creating Azure container...');
         await containerClient.create();
-        console.log("Azure container created successfully");
+        console.log('Azure container created successfully');
     } catch (error) {
-        // Ignore if container already exists
+    // Ignore if container already exists
         if (error.statusCode === 409) {
-            console.log("Azure container already exists");
+            console.log('Azure container already exists');
         } else {
-            console.error("Error creating Azure container:", error);
+            console.error('Error creating Azure container:', error);
             process.exit(1);
         }
     }
@@ -23,19 +26,19 @@ async function createAzureContainer() {
 async function createGCSBucket() {
     try {
         const storage = new Storage({
-            projectId: "test-project",
-            apiEndpoint: "http://localhost:4443",
+            projectId: 'test-project',
+            apiEndpoint: 'http://localhost:4443',
         });
-        
-        console.log("Creating GCS bucket...");
-        await storage.createBucket("cortextempfiles");
-        console.log("GCS bucket created successfully");
+
+        console.log('Creating GCS bucket...');
+        await storage.createBucket('cortextempfiles');
+        console.log('GCS bucket created successfully');
     } catch (error) {
-        // Ignore if bucket already exists
+    // Ignore if bucket already exists
         if (error.code === 409) {
-            console.log("GCS bucket already exists");
+            console.log('GCS bucket already exists');
         } else {
-            console.error("Error creating GCS bucket:", error);
+            console.error('Error creating GCS bucket:', error);
             process.exit(1);
         }
     }
@@ -46,4 +49,4 @@ async function setup() {
     await createGCSBucket();
 }
 
-setup(); 
+setup();

package/helper-apps/cortex-file-handler/scripts/test-azure.sh

@@ -6,7 +6,7 @@ mkdir -p $AZURITE_DIR
 
 # Start Azurite in background
 echo "Starting Azurite..."
-azurite --silent --location $AZURITE_DIR &
+azurite --silent --skipApiVersionCheck --location $AZURITE_DIR &
 AZURITE_PID=$!
 
 # Wait for Azurite to start

package/helper-apps/cortex-file-handler/scripts/test-gcs.sh

@@ -20,7 +20,7 @@ echo "Starting test environment..."
 # Start Azurite if not running
 if ! nc -z localhost 10000; then
     echo "Starting Azurite..."
-    azurite --silent --location .azurite --debug .azurite/debug.log &
+    azurite --silent --skipApiVersionCheck --location .azurite --debug .azurite/debug.log &
     AZURITE_PID=$!
     # Wait for Azurite to be ready
     until nc -z localhost 10000; do

package/helper-apps/cortex-file-handler/services/ConversionService.js

@@ -0,0 +1,288 @@
+import fs from 'fs/promises';
+import os from 'os';
+import path from 'path';
+import { createReadStream, createWriteStream } from 'fs';
+import { pipeline } from 'stream/promises';
+import axios from 'axios';
+import XLSX from 'xlsx';
+import { CONVERTED_EXTENSIONS } from '../constants.js';
+import { v4 as uuidv4 } from 'uuid';
+
+const MARKITDOWN_CONVERT_URL = process.env.MARKITDOWN_CONVERT_URL;
+
+if (!MARKITDOWN_CONVERT_URL) {
+    throw new Error('MARKITDOWN_CONVERT_URL is not set');
+}
+
+export class ConversionService {
+    constructor(context) {
+        this.context = context;
+    }
+
+    /**
+     * Determines if a file needs conversion based on its extension
+     * @param {string} filename - The name of the file to check
+     * @returns {boolean} - Whether the file needs conversion
+     */
+    needsConversion(filename) {
+        const ext = path.extname(filename).toLowerCase();
+        return CONVERTED_EXTENSIONS.includes(ext);
+    }
+
+    /**
+     * Converts a file to its appropriate format
+     * @param {string} filePath - Path to the file to convert
+     * @param {string} originalUrl - Original URL of the file (required for document conversion)
+     * @param {boolean} forceConversion - If true, bypasses extension check and forces document conversion
+     * @returns {Promise<{convertedPath: string, convertedName: string, converted: boolean}>}
+     */
+    async convertFile(filePath, originalUrl = null, forceConversion = false) {
+        this.context.log('Converting file:', { filePath, originalUrl, forceConversion });
+        
+        // Clean the file path by removing any query parameters
+        const cleanFilePath = filePath.split('?')[0];
+        const ext = path.extname(cleanFilePath).toLowerCase();
+        const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'convert-'));
+        
+        try {
+            // If forceConversion is true, directly handle as document conversion
+            if (forceConversion) {
+                return await this._handleDocumentConversion(filePath, originalUrl, tempDir);
+            }
+
+            // Handle Excel files
+            if (ext === '.xlsx' || ext === '.xls') {
+                return await this._handleExcelConversion(filePath, tempDir);
+            }
+
+            // Handle documents that need markdown conversion
+            if (['.docx', '.doc', '.ppt', '.pptx'].includes(ext)) {
+                return await this._handleDocumentConversion(filePath, originalUrl, tempDir);
+            }
+
+            this.context.log('No conversion needed for this file type');
+            return { converted: false };
+        } catch (error) {
+            this.context.log('Error in convertFile:', error);
+            // Clean up temp directory on error
+            await fs.rm(tempDir, { recursive: true, force: true });
+            throw error;
+        }
+    }
+
+    /**
+     * Ensures a file has both original and converted versions
+     * @param {Object} fileInfo - Information about the file
+     * @param {string} requestId - Request ID for storage
+     * @returns {Promise<Object>} - Updated file info with conversion if needed
+     */
+    async ensureConvertedVersion(fileInfo, requestId) {
+        const { url, gcs } = fileInfo;
+        const extension = path.extname(url).toLowerCase();
+        
+        // If file doesn't need conversion, return original info
+        if (!this.needsConversion(extension)) {
+            return fileInfo;
+        }
+
+        // Check if converted version exists in the hash map
+        const convertedKey = `${fileInfo.hash}_converted`;
+        const convertedInfo = await this._getFileStoreMap(convertedKey);
+        
+        let needsConversion = false;
+        if (convertedInfo) {
+            // Verify both primary and GCS URLs exist
+            const primaryExists = await this._urlExists(convertedInfo?.url);
+            const gcsExists = gcs ? await this._gcsUrlExists(convertedInfo?.gcs) : false;
+            
+            // If both URLs exist, return the info
+            if (primaryExists.valid && (!gcs || gcsExists)) {
+                return { ...fileInfo, converted: convertedInfo };
+            }
+            
+            // If either URL is missing, we need to convert
+            needsConversion = true;
+            this.context.log('Conversion needed - missing URLs:', {
+                primaryExists: primaryExists.valid,
+                gcsExists,
+                convertedInfo
+            });
+        } else {
+            needsConversion = true;
+            this.context.log('Conversion needed - no converted info in map');
+        }
+
+        // If conversion is needed, create it
+        if (needsConversion) {
+            try {
+                const tempDir = path.join(os.tmpdir(), `${uuidv4()}`);
+                await fs.mkdir(tempDir);
+                
+                // Download original file
+                const downloadedFile = path.join(tempDir, path.basename(url));
+                await this._downloadFile(url, downloadedFile);
+
+                // Convert the file
+                const conversion = await this.convertFile(downloadedFile, url);
+
+                if (!conversion.converted) {
+                    throw new Error('File conversion failed');
+                }
+
+                // Save converted file to primary storage
+                const convertedSaveResult = await this._saveConvertedFile(conversion.convertedPath, requestId);
+                if (!convertedSaveResult) {
+                    throw new Error('Failed to save converted file to primary storage');
+                }
+
+                // If GCS is configured, also save to GCS
+                let gcsUrl;
+                if (this._isGCSConfigured()) {
+                    gcsUrl = await this._uploadChunkToGCS(conversion.convertedPath, requestId);
+                }
+
+                // Store converted file info
+                const convertedFileInfo = {
+                    url: convertedSaveResult.url,
+                    gcs: gcsUrl
+                };
+
+                // Only store in map if we have at least the primary URL
+                if (convertedFileInfo.url) {
+                    await this._setFileStoreMap(convertedKey, convertedFileInfo);
+                    this.context.log('Stored converted file info:', convertedFileInfo);
+                } else {
+                    throw new Error('Failed to get primary URL for converted file');
+                }
+
+                // Cleanup temp files
+                await this._cleanupTempFiles(downloadedFile, conversion.convertedPath, tempDir);
+
+                return { ...fileInfo, converted: convertedFileInfo };
+            } catch (error) {
+                this.context.log('Error ensuring converted version:', error);
+                // Don't return partial conversion results
+                return fileInfo;
+            }
+        }
+
+        return fileInfo;
+    }
+
+    // Private helper methods
+    async _handleExcelConversion(filePath, tempDir) {
+        this.context.log('Handling Excel file conversion');
+        const csvPath = await this._xlsxToCsv(filePath);
+        const ext = path.extname(filePath);
+        const convertedPath = path.join(
+            tempDir,
+            `${path.basename(filePath, ext)}.csv`,
+        );
+        
+        await pipeline(
+            createReadStream(csvPath, { highWaterMark: 64 * 1024 }),
+            createWriteStream(convertedPath, { highWaterMark: 64 * 1024 }),
+        );
+        await fs.unlink(csvPath);
+        
+        return {
+            convertedPath,
+            convertedName: path.basename(convertedPath),
+            converted: true,
+        };
+    }
+
+    async _handleDocumentConversion(filePath, originalUrl, tempDir) {
+        this.context.log('Handling document conversion');
+        if (!originalUrl) {
+            throw new Error('Original URL is required for document conversion');
+        }
+
+        const markdown = await this._convertToMarkdown(originalUrl);
+        if (!markdown) {
+            throw new Error('Markdown conversion returned empty result');
+        }
+
+        const ext = path.extname(filePath);
+        const baseFilename = path.basename(filePath, ext);
+        const convertedPath = path.join(tempDir, `${baseFilename}.md`);
+        await fs.writeFile(convertedPath, markdown);
+        
+        return {
+            convertedPath,
+            convertedName: path.basename(convertedPath),
+            converted: true,
+        };
+    }
+
+    async _convertToMarkdown(fileUrl) {
+        try {
+            const apiUrl = `${MARKITDOWN_CONVERT_URL}${encodeURIComponent(fileUrl)}`;
+            const response = await axios.get(apiUrl);
+            return response.data.markdown || '';
+        } catch (err) {
+            this.context.log('Error converting to markdown:', err);
+            throw err;
+        }
+    }
+
+    async _xlsxToCsv(filePath) {
+        const workbook = XLSX.readFile(filePath, { type: 'buffer' });
+        const outputPath = filePath.replace(/\.[^/.]+$/, '.csv');
+        let csvContent = '';
+
+        workbook.SheetNames.forEach((sheetName) => {
+            const sheet = workbook.Sheets[sheetName];
+            const csv = XLSX.utils.sheet_to_csv(sheet);
+            csvContent += `Sheet: ${sheetName}\n${csv}\n\n`;
+        });
+
+        await fs.writeFile(outputPath, csvContent);
+        return outputPath;
+    }
+
+    // Storage-related methods (to be implemented by the caller)
+    async _getFileStoreMap(key) {
+        throw new Error('Method _getFileStoreMap must be implemented');
+    }
+
+    async _setFileStoreMap(key, value) {
+        throw new Error('Method _setFileStoreMap must be implemented');
+    }
+
+    async _urlExists(url) {
+        throw new Error('Method _urlExists must be implemented');
+    }
+
+    async _gcsUrlExists(url) {
+        throw new Error('Method _gcsUrlExists must be implemented');
+    }
+
+    async _downloadFile(url, destination) {
+        throw new Error('Method _downloadFile must be implemented');
+    }
+
+    async _saveConvertedFile(filePath, requestId) {
+        throw new Error('Method _saveConvertedFile must be implemented');
+    }
+
+    async _uploadChunkToGCS(filePath, requestId) {
+        throw new Error('Method _uploadChunkToGCS must be implemented');
+    }
+
+    _isGCSConfigured() {
+        throw new Error('Method _isGCSConfigured must be implemented');
+    }
+
+    async _cleanupTempFiles(...files) {
+        for (const file of files) {
+            try {
+                if (file && await fs.access(file).then(() => true).catch(() => false)) {
+                    await fs.unlink(file);
+                }
+            } catch (err) {
+                this.context.log('Error cleaning up temp file:', err);
+            }
+        }
+    }
+} 

package/helper-apps/cortex-file-handler/services/FileConversionService.js

@@ -0,0 +1,53 @@
+import { ConversionService } from './ConversionService.js';
+import { getFileStoreMap, setFileStoreMap } from '../redis.js';
+import { urlExists } from '../helper.js';
+import { gcsUrlExists, uploadChunkToGCS, gcs } from '../blobHandler.js';
+import { downloadFile } from '../fileChunker.js';
+import { saveFileToBlob } from '../blobHandler.js';
+import { moveFileToPublicFolder } from '../localFileHandler.js';
+
+export class FileConversionService extends ConversionService {
+    constructor(context, useAzure = true) {
+        super(context);
+        this.useAzure = useAzure;
+    }
+
+    async _getFileStoreMap(key) {
+        return getFileStoreMap(key);
+    }
+
+    async _setFileStoreMap(key, value) {
+        return setFileStoreMap(key, value);
+    }
+
+    async _urlExists(url) {
+        return urlExists(url);
+    }
+
+    async _gcsUrlExists(url) {
+        return gcsUrlExists(url);
+    }
+
+    async _downloadFile(url, destination) {
+        return downloadFile(url, destination);
+    }
+
+    async _saveConvertedFile(filePath, requestId) {
+        let fileUrl;    
+        if (this.useAzure) {
+            const savedBlob = await saveFileToBlob(filePath, requestId);
+            fileUrl = savedBlob.url;
+        } else {
+            fileUrl = await moveFileToPublicFolder(filePath, requestId);
+        }
+        return { url: fileUrl };
+    }
+
+    async _uploadChunkToGCS(filePath, requestId) {
+        return uploadChunkToGCS(filePath, requestId);
+    }
+
+    _isGCSConfigured() {
+        return !!gcs;
+    }
+}