@aj-archipelago/cortex 1.3.51 → 1.3.53

package/helper-apps/cortex-file-handler/tests/conversionResilience.test.js

@@ -0,0 +1,152 @@
+import test from 'ava';
+import axios from 'axios';
+import fs from 'fs';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+import XLSX from 'xlsx';
+
+import { port } from '../src/start.js';
+import { gcs, GCS_BUCKETNAME } from '../src/blobHandler.js';
+import { getFileStoreMap, setFileStoreMap } from '../src/redis.js';
+import { cleanupHashAndFile } from './testUtils.helper.js';
+import { gcsUrlExists } from '../src/blobHandler.js';
+
+const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
+
+// helper: create in-memory xlsx -> file
+async function createXlsx(tmpDir) {
+    const wb = XLSX.utils.book_new();
+    const ws = XLSX.utils.aoa_to_sheet([
+        ['A', 'B'],
+        ['1', '2'],
+    ]);
+    XLSX.utils.book_append_sheet(wb, ws, 'Sheet1');
+    const filePath = path.join(tmpDir, `${uuidv4()}.xlsx`);
+    XLSX.writeFile(wb, filePath);
+    return filePath;
+}
+
+// Upload helper (multipart)
+async function multipartUpload(filePath, hash) {
+    const FormData = (await import('form-data')).default;
+    const form = new FormData();
+    form.append('hash', hash);
+    form.append('file', fs.createReadStream(filePath));
+
+    const res = await axios.post(baseUrl, form, {
+        headers: form.getHeaders(),
+        validateStatus: () => true,
+        timeout: 30000,
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+    });
+    return res;
+}
+
+function isGcsConfigured() {
+    return !!gcs;
+}
+
+test.before(async t => {
+    const dir = path.join(fs.mkdtempSync(path.join(process.cwd(), 'conv-test-')));
+    t.context.tmpDir = dir;
+});
+
+test.after.always(async t => {
+    fs.rmSync(t.context.tmpDir, { recursive: true, force: true });
+});
+
+// 1. Remote-URL upload path should still return converted info
+
+test.serial('remote URL save returns converted info', async t => {
+    const filePath = await createXlsx(t.context.tmpDir);
+    const hash = `hash-${uuidv4()}`;
+    // step 1: multipart upload
+    const up = await multipartUpload(filePath, hash);
+    t.is(up.status, 200);
+    t.truthy(up.data.converted?.url);
+    const publicUrl = up.data.url;
+
+    // step 2: call handler via ?uri= <publicUrl>&save=true
+    const saveRes = await axios.get(baseUrl, {
+        params: {
+            uri: publicUrl,
+            requestId: uuidv4(),
+            save: true,
+        },
+        validateStatus: () => true,
+        timeout: 30000,
+    });
+
+    t.is(saveRes.status, 200);
+
+    t.true(saveRes.data?.url?.includes('.csv'));
+
+    await cleanupHashAndFile(hash, up.data.url, baseUrl);
+});
+
+// 2. If converted.gcs is missing, checkHash should restore it
+
+test.serial('checkHash recreates missing GCS converted file', async t => {
+    if (!isGcsConfigured()) {
+        t.pass();
+        return;
+    }
+
+    const filePath = await createXlsx(t.context.tmpDir);
+    const hash = `hash-${uuidv4()}`;
+    const up = await multipartUpload(filePath, hash);
+    t.truthy(up.data.converted?.gcs);
+
+    // delete the GCS object
+    const convertedGcsUrl = up.data.converted.gcs;
+    const bucket = gcs.bucket(GCS_BUCKETNAME);
+    const filename = convertedGcsUrl.replace(`gs://${GCS_BUCKETNAME}/`, '');
+    try {
+        await bucket.file(filename).delete({ ignoreNotFound: true });
+    } catch (_) {}
+
+    // call checkHash – should restore
+    const resp = await axios.get(baseUrl, {
+        params: { hash, checkHash: true },
+        validateStatus: () => true,
+        timeout: 30000,
+    });
+    t.is(resp.status, 200);
+    t.truthy(resp.data.converted?.gcs);
+
+    // verify restored GCS object exists using returned URL
+    const newGcsUrl = resp.data.converted.gcs;
+    const existsAfter = await gcsUrlExists(newGcsUrl, false);
+    t.true(existsAfter);
+
+    await cleanupHashAndFile(hash, up.data.url, baseUrl);
+});
+
+// 3. If converted section is removed from Redis, checkHash regenerates
+
+test.serial('checkHash regenerates missing converted metadata', async t => {
+    const filePath = await createXlsx(t.context.tmpDir);
+    const hash = `hash-${uuidv4()}`;
+    const up = await multipartUpload(filePath, hash);
+    t.truthy(up.data.converted?.url);
+
+    // strip converted from Redis entry
+    const record = await getFileStoreMap(hash);
+    if (record) {
+        delete record.converted;
+        await setFileStoreMap(hash, record);
+    }
+
+    // call checkHash – should add converted back
+    const resp = await axios.get(baseUrl, {
+        params: { hash, checkHash: true },
+        validateStatus: () => true,
+        timeout: 30000,
+    });
+
+    t.is(resp.status, 200);
+    t.truthy(resp.data.converted?.url);
+
+    await cleanupHashAndFile(hash, up.data.url, baseUrl);
+}); 

package/helper-apps/cortex-file-handler/tests/fileChunker.test.js

@@ -9,37 +9,11 @@ import { fileURLToPath } from 'url';
 import test from 'ava';
 import nock from 'nock';
 
-import { splitMediaFile, downloadFile } from '../fileChunker.js';
+import { splitMediaFile, downloadFile } from '../src/fileChunker.js';
+import { createTestMediaFile } from './testUtils.helper.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-// Helper function to create a test media file of specified duration using ffmpeg
-async function createTestMediaFile(filepath, durationSeconds = 10) {
-    try {
-        console.log(`Creating test file: ${filepath} (${durationSeconds}s)`);
-        // Generate silence using ffmpeg
-        execSync(
-            `ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t ${durationSeconds} -q:a 9 -acodec libmp3lame "${filepath}"`,
-            {
-                stdio: ['ignore', 'pipe', 'pipe'], // Capture stdout and stderr
-            },
-        );
-
-        // Verify the file was created and has content
-        const stats = await fs.stat(filepath);
-        if (stats.size === 0) {
-            throw new Error('Generated file is empty');
-        }
-        console.log(
-            `Successfully created ${filepath} (${(stats.size / 1024 / 1024).toFixed(2)}MB)`,
-        );
-    } catch (error) {
-        console.error(`Error creating test file ${filepath}:`, error.message);
-        if (error.stderr) console.error('ffmpeg error:', error.stderr.toString());
-        throw error;
-    }
-}
-
 // Setup: Create test files and mock external services
 test.before(async (t) => {
     // Check if ffmpeg is available

package/helper-apps/cortex-file-handler/tests/fileUpload.test.js

@@ -5,9 +5,10 @@ import { fileURLToPath } from 'url';
 import { v4 as uuidv4 } from 'uuid';
 import axios from 'axios';
 import FormData from 'form-data';
-import { port } from '../start.js';
-import { gcs } from '../blobHandler.js';
-import { cleanupHashAndFile } from './testUtils.helper.js';
+import { port } from '../src/start.js';
+import { gcs } from '../src/blobHandler.js';
+import { cleanupHashAndFile, getFolderNameFromUrl } from './testUtils.helper.js';
+import XLSX from 'xlsx';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -45,7 +46,7 @@ async function uploadFile(filePath, requestId = null, hash = null) {
             'Content-Type': 'multipart/form-data',
         },
         validateStatus: (status) => true,
-        timeout: 5000,
+        timeout: 30000,
         maxContentLength: Infinity,
         maxBodyLength: Infinity,
     });
@@ -83,7 +84,25 @@ test.before(async (t) => {
 
 // Cleanup
 test.after.always(async (t) => {
+    // Clean up test directory
     await fs.promises.rm(t.context.testDir, { recursive: true, force: true });
+    
+    // Clean up any remaining files in the files directory
+    const filesDir = path.join(__dirname, '..', 'files');
+    if (fs.existsSync(filesDir)) {
+        const dirs = await fs.promises.readdir(filesDir);
+        for (const dir of dirs) {
+            const dirPath = path.join(filesDir, dir);
+            try {
+                await fs.promises.rm(dirPath, { recursive: true, force: true });
+            } catch (e) {
+                console.error('Error cleaning up directory:', {
+                    dir: dirPath,
+                    error: e.message
+                });
+            }
+        }
+    }
 });
 
 // Basic File Upload Tests
@@ -91,9 +110,10 @@ test.serial('should handle basic file upload', async (t) => {
     const fileContent = 'test content';
     const filePath = await createTestFile(fileContent, 'txt');
     const requestId = uuidv4();
+    let response;
     
     try {
-        const response = await uploadFile(filePath, requestId);
+        response = await uploadFile(filePath, requestId);
         
         t.is(response.status, 200);
         t.truthy(response.data.url);
@@ -104,6 +124,9 @@ test.serial('should handle basic file upload', async (t) => {
         t.deepEqual(uploadedContent, Buffer.from(fileContent), 'Uploaded file content should match');
     } finally {
         fs.unlinkSync(filePath);
+        if (response?.data?.url) {
+            await cleanupHashAndFile(null, response.data.url, baseUrl);
+        }
     }
 });
 
@@ -114,9 +137,11 @@ test.serial('should handle file upload with hash', async (t) => {
     const hash = 'test-hash-' + uuidv4();
     let uploadedUrl;
     let convertedUrl;
+    let response;
+    
     try {
         // First upload the file
-        const response = await uploadFile(filePath, requestId, hash);
+        response = await uploadFile(filePath, requestId, hash);
         t.is(response.status, 200);
         t.truthy(response.data.url);
         uploadedUrl = response.data.url;
@@ -153,9 +178,11 @@ test.serial('should handle file upload with hash', async (t) => {
         t.deepEqual(Buffer.from(fileResponse.data), Buffer.from(fileContent), 'Uploaded file content should match');
     } finally {
         fs.unlinkSync(filePath);
-        await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
+        if (uploadedUrl) {
+            await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
+        }
         if (convertedUrl) {
-            await cleanupHashAndFile(`${hash}_converted`, convertedUrl, baseUrl);
+            await cleanupHashAndFile(null, convertedUrl, baseUrl);
         }
     }
 });
@@ -166,9 +193,10 @@ test.serial('should handle PDF document upload and conversion', async (t) => {
     const fileContent = '%PDF-1.4\nTest PDF content';
     const filePath = await createTestFile(fileContent, 'pdf');
     const requestId = uuidv4();
+    let response;
     
     try {
-        const response = await uploadFile(filePath, requestId);
+        response = await uploadFile(filePath, requestId);
         t.is(response.status, 200);
         t.truthy(response.data.url);
 
@@ -186,6 +214,12 @@ test.serial('should handle PDF document upload and conversion', async (t) => {
         }
     } finally {
         fs.unlinkSync(filePath);
+        if (response?.data?.url) {
+            await cleanupHashAndFile(null, response.data.url, baseUrl);
+        }
+        if (response?.data?.converted?.url) {
+            await cleanupHashAndFile(null, response.data.converted.url, baseUrl);
+        }
     }
 });
 
@@ -195,9 +229,10 @@ test.serial('should handle media file chunking', async (t) => {
     const chunkContent = 'x'.repeat(1024 * 1024);
     const filePath = await createTestFile(chunkContent, 'mp4');
     const requestId = uuidv4();
+    let response;
     
     try {
-        const response = await uploadFile(filePath, requestId);
+        response = await uploadFile(filePath, requestId);
         t.is(response.status, 200);
         t.truthy(response.data);
         
@@ -232,6 +267,17 @@ test.serial('should handle media file chunking', async (t) => {
         }
     } finally {
         fs.unlinkSync(filePath);
+        if (response?.data) {
+            if (Array.isArray(response.data)) {
+                for (const chunk of response.data) {
+                    if (chunk.uri) {
+                        await cleanupHashAndFile(null, chunk.uri, baseUrl);
+                    }
+                }
+            } else if (response.data.url) {
+                await cleanupHashAndFile(null, response.data.url, baseUrl);
+            }
+        }
     }
 });
 
@@ -249,7 +295,7 @@ test.serial('should handle invalid file upload', async (t) => {
             'Content-Type': 'multipart/form-data',
         },
         validateStatus: (status) => true,
-        timeout: 5000,
+        timeout: 30000,
     });
     
     // Log the response for debugging
@@ -276,8 +322,7 @@ test.serial('should handle file deletion', async (t) => {
         await new Promise(resolve => setTimeout(resolve, 1000));
         
         // Extract the file identifier from the URL
-        const fileUrl = new URL(uploadResponse.data.url);
-        const fileIdentifier = fileUrl.pathname.split('/').pop().split('_')[0];
+        const fileIdentifier = getFolderNameFromUrl(uploadResponse.data.url);
         console.log('File identifier for deletion:', fileIdentifier);
         
         // Delete file using the correct identifier
@@ -307,13 +352,24 @@ test.serial('should handle file deletion', async (t) => {
 
 // Save Option Test
 test.serial('should handle document upload with save option', async (t) => {
-    const fileContent = 'Sample DOCX content';
-    const filePath = await createTestFile(fileContent, 'docx');
+    // Create a minimal XLSX workbook in-memory
+    const workbook = XLSX.utils.book_new();
+    const worksheet = XLSX.utils.aoa_to_sheet([
+        ['Name', 'Score'],
+        ['Alice', 10],
+        ['Bob', 8],
+    ]);
+    XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1');
+
+    // Write it to a temp file inside the test directory
+    const filePath = path.join(t.context.testDir, `${uuidv4()}.xlsx`);
+    XLSX.writeFile(workbook, filePath);
 
     const initialRequestId = uuidv4();
     const saveRequestId = uuidv4();
 
     let uploadedUrl;
+    let savedUrl;
 
     try {
         // First, upload the document so we have a publicly reachable URL
@@ -333,19 +389,74 @@ test.serial('should handle document upload with save option', async (t) => {
             validateStatus: (status) => true,
         });
 
-        // The current implementation returns an empty array but should still be 200
+        // The save operation should return a 200 status with a result object
         t.is(saveResponse.status, 200, 'Save request should succeed');
-        t.true(Array.isArray(saveResponse.data), 'Response body should be an array');
+        t.truthy(saveResponse.data, 'Response should have data');
+        t.truthy(saveResponse.data.url, 'Response should include a URL');
+        t.true(saveResponse.data.url.includes('.csv'), 'Response should include a CSV URL');
+        savedUrl = saveResponse.data.url;
     } finally {
         fs.unlinkSync(filePath);
-
-        // Cleanup the initially uploaded file
+        // Clean up both URLs
         if (uploadedUrl) {
-            const identifier = new URL(uploadedUrl).pathname.split('/').pop().split('_')[0];
-            await axios.delete(`${baseUrl}?operation=delete&requestId=${identifier}`).catch(() => {});
+            await cleanupHashAndFile(null, uploadedUrl, baseUrl);
+        }
+        if (savedUrl && savedUrl !== uploadedUrl) {
+            await cleanupHashAndFile(null, savedUrl, baseUrl);
         }
+    }
+});
+
+// Converted file persistence test – ensures needsConversion works for extension-only checks
+test.serial('should preserve converted version when checking hash for convertible file', async (t) => {
+    // Create a minimal XLSX workbook in-memory
+    const workbook = XLSX.utils.book_new();
+    const worksheet = XLSX.utils.aoa_to_sheet([
+        ['Name', 'Score'],
+        ['Alice', 10],
+        ['Bob', 8],
+    ]);
+    XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1');
 
-        // Cleanup files created by save request
-        await axios.delete(`${baseUrl}?operation=delete&requestId=${saveRequestId}`).catch(() => {});
+    // Write it to a temp file inside the test directory
+    const filePath = path.join(t.context.testDir, `${uuidv4()}.xlsx`);
+    XLSX.writeFile(workbook, filePath);
+
+    const requestId = uuidv4();
+    const hash = `test-hash-${uuidv4()}`;
+
+    let uploadedUrl;
+    let convertedUrl;
+
+    try {
+        // 1. Upload the XLSX file (conversion should run automatically)
+        const uploadResponse = await uploadFile(filePath, requestId, hash);
+        t.is(uploadResponse.status, 200, 'Upload should succeed');
+        t.truthy(uploadResponse.data.converted, 'Upload response must contain converted info');
+        t.truthy(uploadResponse.data.converted.url, 'Converted URL should be present');
+
+        uploadedUrl = uploadResponse.data.url;
+        convertedUrl = uploadResponse.data.converted.url;
+
+        // 2. Give Redis a moment to persist
+        await new Promise((resolve) => setTimeout(resolve, 4000));
+
+        // 3. Ask the handler for the hash – it will invoke ensureConvertedVersion
+        const checkResponse = await axios.get(baseUrl, {
+            params: { hash, checkHash: true },
+            validateStatus: (status) => true,
+            timeout: 30000,
+        });
+
+        t.is(checkResponse.status, 200, 'Hash check should succeed');
+        t.truthy(checkResponse.data.converted, 'Hash response should include converted info');
+        t.truthy(checkResponse.data.converted.url, 'Converted URL should still be present after hash check');
+    } finally {
+        // Clean up temp file and remote artifacts
+        fs.unlinkSync(filePath);
+        await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
+        if (convertedUrl) {
+            await cleanupHashAndFile(null, convertedUrl, baseUrl);
+        }
     }
 });