@aj-archipelago/cortex 1.3.51 → 1.3.52

package/helper-apps/cortex-file-handler/tests/start.test.js

@@ -11,7 +11,7 @@ import axios from 'axios';
 import FormData from 'form-data';
 import { v4 as uuidv4 } from 'uuid';
 
-import { port, publicFolder, ipAddress } from '../start.js';
+import { port, publicFolder, ipAddress } from '../src/start.js';
 import { cleanupHashAndFile, getFolderNameFromUrl } from './testUtils.helper.js';
 
 // Add these helper functions at the top after imports
@@ -259,10 +259,11 @@ test.serial('should reject invalid URLs', async (t) => {
         },
     );
 
-    t.is(response.status, 500, 'Should return 500 for invalid URL');
-    t.true(
-        response.data.includes('Invalid URL'),
-        'Should indicate invalid URL in error message',
+    t.is(response.status, 400, 'Should return 400 for invalid URL');
+    t.is(
+        response.data,
+        'Invalid URL format',
+        'Should indicate invalid URL format in error message',
     );
 });
 
@@ -279,10 +280,11 @@ test.serial('should reject unsupported protocols', async (t) => {
         },
     );
 
-    t.is(response.status, 500, 'Should return 500 for unsupported protocol');
-    t.true(
-        response.data.includes('Error processing media file'),
-        'Should indicate error processing media file',
+    t.is(response.status, 400, 'Should return 400 for unsupported protocol');
+    t.is(
+        response.data,
+        'Invalid URL protocol - only HTTP, HTTPS, and GCS URLs are supported',
+        'Should indicate invalid protocol in error message',
     );
 });
 
@@ -668,7 +670,7 @@ test.serial(
         t.is(uploadResponse.status, 200, 'Upload should succeed');
         t.truthy(uploadResponse.data.url, 'Response should contain URL');
 
-        await cleanupUploadedFile(t, uploadResponse.data.url);
+        await cleanupHashAndFile(testHash, uploadResponse.data.url, baseUrl);
 
         // Verify hash is gone by trying to get the file URL
         const hashCheckResponse = await axios.get(`${baseUrl}`, {
@@ -835,13 +837,19 @@ async function checkGCSFile(gcsUrl) {
     const [, , bucket, ...objectParts] = gcsUrl.split('/');
     const object = objectParts.join('/');
 
+    console.log(`[checkGCSFile] Checking file in GCS: ${gcsUrl}`);
+    console.log(`[checkGCSFile] Bucket: ${bucket}, Object: ${object}`);
+    console.log(`[checkGCSFile] Using emulator at ${process.env.STORAGE_EMULATOR_HOST}`);
+
     // Query fake-gcs-server
     const response = await axios.get(
-        `http://localhost:4443/storage/v1/b/${bucket}/o/${encodeURIComponent(object)}`,
+        `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${bucket}/o/${encodeURIComponent(object)}`,
         {
             validateStatus: (status) => true,
         },
     );
+    console.log(`[checkGCSFile] Response status: ${response.status}`);
+    console.log(`[checkGCSFile] File ${response.status === 200 ? 'exists' : 'does not exist'}`);
     return response.status === 200;
 }
 
@@ -1031,6 +1039,7 @@ test.serial(
             params: {
                 uri: testFileUrl,
                 requestId,
+                useGCS: true,
             },
             validateStatus: (status) => true,
             timeout: 5000,
@@ -1327,13 +1336,14 @@ test.serial(
     async (t) => {
         const testContent = 'test content for legacy sequence';
         const testHash = 'test-legacy-sequence-hash';
+        const legacyBaseUrl = `http://localhost:${port}/api/MediaFileChunker`;
         const form = new FormData();
         form.append('file', Buffer.from(testContent), 'sequence-test.txt');
         form.append('hash', testHash);
 
         // Upload file with hash through legacy endpoint
         const uploadResponse = await axios.post(
-            `http://localhost:${port}/api/MediaFileChunker`,
+            legacyBaseUrl,
             form,
             {
                 headers: form.getHeaders(),
@@ -1348,17 +1358,43 @@ test.serial(
         );
         t.truthy(uploadResponse.data.url, 'Response should contain URL');
 
-        await cleanupUploadedFile(t, uploadResponse.data.url);
+        // Wait for Redis to be updated after upload
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+
+        // Verify hash exists after upload using legacy endpoint
+        const initialHashCheck = await axios.get(
+            legacyBaseUrl,
+            {
+                params: {
+                    hash: testHash,
+                    checkHash: true,
+                },
+                validateStatus: (status) => true,
+                timeout: 5000,
+            },
+        );
+        t.is(initialHashCheck.status, 200, 'Hash should exist after upload');
+        t.truthy(initialHashCheck.data.url, 'Hash check should return file URL');
+
+        // Wait for Redis to be updated after initial check
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+
+        // Clean up the file and hash using the legacy endpoint
+        await cleanupHashAndFile(testHash, uploadResponse.data.url, legacyBaseUrl);
+
+        // Wait for Redis to be updated after cleanup
+        await new Promise((resolve) => setTimeout(resolve, 1000));
 
         // Verify hash is gone by trying to get the file URL through legacy endpoint
         const hashCheckResponse = await axios.get(
-            `http://localhost:${port}/api/MediaFileChunker`,
+            legacyBaseUrl,
             {
                 params: {
                     hash: testHash,
                     checkHash: true,
                 },
                 validateStatus: (status) => true,
+                timeout: 5000,
             },
         );
         t.is(hashCheckResponse.status, 404, 'Hash should not exist after deletion');

package/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js

@@ -0,0 +1,120 @@
+import test from 'ava';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { AzureStorageProvider } from '../../src/services/storage/AzureStorageProvider.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+test.before(() => {
+    // Ensure we have the required environment variables
+    if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+        console.warn('Skipping Azure tests - AZURE_STORAGE_CONNECTION_STRING not set');
+    }
+});
+
+test('should create provider with valid credentials', (t) => {
+    if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+        t.pass('Skipping test - Azure not configured');
+        return;
+    }
+
+    const provider = new AzureStorageProvider(
+        process.env.AZURE_STORAGE_CONNECTION_STRING,
+        'test-container'
+    );
+    t.truthy(provider);
+});
+
+test('should throw error with missing credentials', (t) => {
+    t.throws(() => {
+        new AzureStorageProvider(null, 'test-container');
+    }, { message: 'Missing Azure Storage connection string or container name' });
+});
+
+test('should upload and delete file', async (t) => {
+    if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+        t.pass('Skipping test - Azure not configured');
+        return;
+    }
+
+    const provider = new AzureStorageProvider(
+        process.env.AZURE_STORAGE_CONNECTION_STRING,
+        'test-container'
+    );
+
+    // Create test file
+    const testContent = 'Hello World!';
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+
+    try {
+        // Upload file
+        const requestId = 'test-upload';
+        const result = await provider.uploadFile({}, testFile, requestId);
+        
+        t.truthy(result.url);
+        t.truthy(result.blobName);
+        t.true(result.url.includes('test-container'));
+        t.true(result.blobName.startsWith(requestId));
+
+        // Verify file exists
+        const exists = await provider.fileExists(result.url);
+        t.true(exists);
+
+        // Delete file
+        const deleted = await provider.deleteFiles(requestId);
+        t.true(deleted.length > 0);
+        t.true(deleted[0].startsWith(requestId));
+
+        // Verify file is gone
+        const existsAfterDelete = await provider.fileExists(result.url);
+        t.false(existsAfterDelete);
+    } finally {
+        // Cleanup test file
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('should handle file download', async (t) => {
+    if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+        t.pass('Skipping test - Azure not configured');
+        return;
+    }
+
+    const provider = new AzureStorageProvider(
+        process.env.AZURE_STORAGE_CONNECTION_STRING,
+        'test-container'
+    );
+
+    // Create test file
+    const testContent = 'Hello World!';
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+
+    try {
+        // Upload file
+        const requestId = 'test-download';
+        const result = await provider.uploadFile({}, testFile, requestId);
+
+        // Download to new location
+        const downloadPath = path.join(__dirname, 'downloaded.txt');
+        await provider.downloadFile(result.url, downloadPath);
+
+        // Verify content
+        const downloadedContent = fs.readFileSync(downloadPath, 'utf8');
+        t.is(downloadedContent, testContent);
+
+        // Cleanup
+        await provider.deleteFiles(requestId);
+        fs.unlinkSync(downloadPath);
+    } finally {
+        // Cleanup test file
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+}); 

package/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js

@@ -0,0 +1,193 @@
+import test from 'ava';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { GCSStorageProvider } from '../../src/services/storage/GCSStorageProvider.js';
+import { sanitizeFilename } from '../../src/utils/filenameUtils.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+test.before(() => {
+    // Ensure we have the required environment variables
+    if (!process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && !process.env.GCP_SERVICE_ACCOUNT_KEY) {
+        console.warn('Skipping GCS tests - GCP credentials not set');
+    }
+});
+
+test('should create provider with valid credentials', (t) => {
+    if (!process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && !process.env.GCP_SERVICE_ACCOUNT_KEY) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    const credentials = {
+        project_id: 'test-project',
+        client_email: 'test@test.com',
+        private_key: 'test-key'
+    };
+
+    const provider = new GCSStorageProvider(credentials, 'test-bucket');
+    t.truthy(provider);
+});
+
+test('should throw error with missing credentials', (t) => {
+    t.throws(() => {
+        new GCSStorageProvider(null, 'test-bucket');
+    }, { message: 'Missing GCS credentials or bucket name' });
+});
+
+test('should upload and delete file', async (t) => {
+    if (!process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && !process.env.GCP_SERVICE_ACCOUNT_KEY) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    const credentials = JSON.parse(
+        process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64
+            ? Buffer.from(process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64, 'base64').toString()
+            : process.env.GCP_SERVICE_ACCOUNT_KEY
+    );
+
+    const provider = new GCSStorageProvider(
+        credentials,
+        process.env.GCS_BUCKETNAME || 'cortextempfiles'
+    );
+
+    // Create test file
+    const testContent = 'Hello World!';
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+
+    try {
+        // Upload file
+        const requestId = 'test-upload';
+        const result = await provider.uploadFile({}, testFile, requestId);
+        
+        t.truthy(result.url);
+        t.truthy(result.blobName);
+        t.true(result.url.startsWith('gs://'));
+        t.true(result.blobName.startsWith(requestId));
+
+        // Verify file exists
+        const exists = await provider.fileExists(result.url);
+        t.true(exists);
+
+        // Delete file
+        const deleted = await provider.deleteFiles(requestId);
+        t.true(deleted.length > 0);
+        t.true(deleted[0].startsWith(requestId));
+
+        // Verify file is gone
+        const existsAfterDelete = await provider.fileExists(result.url);
+        t.false(existsAfterDelete);
+    } finally {
+        // Cleanup test file
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('should handle file download', async (t) => {
+    if (!process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && !process.env.GCP_SERVICE_ACCOUNT_KEY) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    const credentials = JSON.parse(
+        process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64
+            ? Buffer.from(process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64, 'base64').toString()
+            : process.env.GCP_SERVICE_ACCOUNT_KEY
+    );
+
+    const provider = new GCSStorageProvider(
+        credentials,
+        process.env.GCS_BUCKETNAME || 'cortextempfiles'
+    );
+
+    // Create test file
+    const testContent = 'Hello World!';
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+
+    try {
+        // Upload file
+        const requestId = 'test-download';
+        const result = await provider.uploadFile({}, testFile, requestId);
+
+        // Download to new location
+        const downloadPath = path.join(__dirname, 'downloaded.txt');
+        await provider.downloadFile(result.url, downloadPath);
+
+        // Verify content
+        const downloadedContent = fs.readFileSync(downloadPath, 'utf8');
+        t.is(downloadedContent, testContent);
+
+        // Cleanup
+        await provider.deleteFiles(requestId);
+        fs.unlinkSync(downloadPath);
+    } finally {
+        // Cleanup test file
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('should handle file existence check with spaces and special characters', async (t) => {
+    if (!process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && !process.env.GCP_SERVICE_ACCOUNT_KEY) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    const credentials = JSON.parse(
+        process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64
+            ? Buffer.from(process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64, 'base64').toString()
+            : process.env.GCP_SERVICE_ACCOUNT_KEY
+    );
+
+    const provider = new GCSStorageProvider(
+        credentials,
+        process.env.GCS_BUCKETNAME || 'cortextempfiles'
+    );
+
+    // Create test file with spaces and special characters in name
+    const testContent = 'Hello World!';
+    const testFileName = 'test file with spaces & special chars!.txt';
+    const testFile = path.join(__dirname, testFileName);
+    fs.writeFileSync(testFile, testContent);
+
+    try {
+        // Upload file
+        const requestId = 'test-special-chars';
+        const result = await provider.uploadFile({}, testFile, requestId);
+        
+        t.truthy(result.url);
+        // Compare against sanitized filename instead of original
+        const sanitizedFileName = sanitizeFilename(testFileName);
+        t.true(result.url.includes(sanitizedFileName));
+        t.true(result.url.startsWith('gs://'));
+
+        // Verify file exists with original URL
+        const exists = await provider.fileExists(result.url);
+        t.true(exists, 'File should exist with original URL');
+
+        // Verify file exists with encoded URL
+        const encodedUrl = result.url.replace(/ /g, '%20');
+        const existsEncoded = await provider.fileExists(encodedUrl);
+        t.true(existsEncoded, 'File should exist with encoded URL');
+
+        // Cleanup
+        await provider.deleteFiles(requestId);
+        
+        // Verify file is gone
+        const existsAfterDelete = await provider.fileExists(result.url);
+        t.false(existsAfterDelete, 'File should not exist after deletion');
+    } finally {
+        // Cleanup test file
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+}); 

package/helper-apps/cortex-file-handler/tests/storage/LocalStorageProvider.test.js

@@ -0,0 +1,148 @@
+import test from 'ava';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { LocalStorageProvider } from '../../src/services/storage/LocalStorageProvider.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+test.before(() => {
+    // Create test directory
+    const testDir = path.join(__dirname, 'test-files');
+    if (!fs.existsSync(testDir)) {
+        fs.mkdirSync(testDir, { recursive: true });
+    }
+});
+
+test.after(() => {
+    // Cleanup test directory
+    const testDir = path.join(__dirname, 'test-files');
+    if (fs.existsSync(testDir)) {
+        fs.rmSync(testDir, { recursive: true });
+    }
+});
+
+test('should create provider with valid path', (t) => {
+    const provider = new LocalStorageProvider(path.join(__dirname, 'test-files'));
+    t.truthy(provider);
+});
+
+test('should throw error with missing path', (t) => {
+    t.throws(() => {
+        new LocalStorageProvider(null);
+    }, { message: 'Missing public folder path' });
+});
+
+test('should create public folder if it does not exist', (t) => {
+    const testDir = path.join(__dirname, 'test-files', 'new-folder');
+    if (fs.existsSync(testDir)) {
+        fs.rmSync(testDir, { recursive: true });
+    }
+
+    const provider = new LocalStorageProvider(testDir);
+    t.true(fs.existsSync(testDir));
+});
+
+test('should upload and delete file', async (t) => {
+    const provider = new LocalStorageProvider(path.join(__dirname, 'test-files'));
+
+    // Create test file
+    const testContent = 'Hello World!';
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+
+    try {
+        // Upload file
+        const requestId = 'test-upload';
+        const result = await provider.uploadFile({}, testFile, requestId);
+        
+        t.truthy(result.url);
+        t.truthy(result.blobName);
+        t.true(result.url.includes('/files/'));
+        t.true(result.blobName.startsWith(requestId));
+
+        // Verify file exists
+        const exists = await provider.fileExists(result.url);
+        t.true(exists);
+
+        // Delete file
+        const deleted = await provider.deleteFiles(requestId);
+        t.true(deleted.length > 0);
+        t.true(deleted[0].startsWith(requestId));
+
+        // Verify file is gone
+        const existsAfterDelete = await provider.fileExists(result.url);
+        t.false(existsAfterDelete);
+    } finally {
+        // Cleanup test file
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('should handle file download', async (t) => {
+    const provider = new LocalStorageProvider(path.join(__dirname, 'test-files'));
+
+    // Create test file
+    const testContent = 'Hello World!';
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+
+    try {
+        // Upload file
+        const requestId = 'test-download';
+        const result = await provider.uploadFile({}, testFile, requestId);
+
+        // Download to new location
+        const downloadPath = path.join(__dirname, 'downloaded.txt');
+        await provider.downloadFile(result.url, downloadPath);
+
+        // Verify content
+        const downloadedContent = fs.readFileSync(downloadPath, 'utf8');
+        t.is(downloadedContent, testContent);
+
+        // Cleanup
+        await provider.deleteFiles(requestId);
+        fs.unlinkSync(downloadPath);
+    } finally {
+        // Cleanup test file
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('should handle cleanup of multiple files', async (t) => {
+    const provider = new LocalStorageProvider(path.join(__dirname, 'test-files'));
+
+    // Create test files
+    const testContent = 'Hello World!';
+    const testFile1 = path.join(__dirname, 'test1.txt');
+    const testFile2 = path.join(__dirname, 'test2.txt');
+    fs.writeFileSync(testFile1, testContent);
+    fs.writeFileSync(testFile2, testContent);
+
+    try {
+        // Upload files
+        const requestId1 = 'test-cleanup-1';
+        const requestId2 = 'test-cleanup-2';
+        const result1 = await provider.uploadFile({}, testFile1, requestId1);
+        const result2 = await provider.uploadFile({}, testFile2, requestId2);
+
+        // Cleanup files
+        const cleaned = await provider.cleanup([result1.url, result2.url]);
+        t.is(cleaned.length, 2);
+
+        // Verify files are gone
+        const exists1 = await provider.fileExists(result1.url);
+        const exists2 = await provider.fileExists(result2.url);
+        t.false(exists1);
+        t.false(exists2);
+    } finally {
+        // Cleanup test files
+        if (fs.existsSync(testFile1)) fs.unlinkSync(testFile1);
+        if (fs.existsSync(testFile2)) fs.unlinkSync(testFile2);
+    }
+});