@aj-archipelago/cortex 1.3.11 → 1.3.14

package/helper-apps/cortex-file-handler/package.json

@@ -1,12 +1,15 @@
 {
-  "name": "@aj-archipelago/mediafilechunker",
-  "version": "1.0.0",
-  "description": "",
+  "name": "@aj-archipelago/cortex-file-handler",
+  "version": "1.0.16",
+  "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "scripts": {
     "start": "node start.js",
     "dev": "node -r dotenv/config start.js",
-    "test": "echo \"No tests yet...\""
+    "test": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs",
+    "test:azure": "DOTENV_CONFIG_PATH=.env.test.azure NODE_ENV=test ./scripts/test-azure.sh",
+    "test:watch": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs --watch",
+    "test:gcs": "DOTENV_CONFIG_PATH=.env.test.gcs NODE_ENV=test ./scripts/test-gcs.sh"
   },
   "dependencies": {
     "@azure/storage-blob": "^12.13.0",
@@ -24,5 +27,23 @@
     "public-ip": "^6.0.1",
     "uuid": "^9.0.0",
     "xlsx": "^0.18.5"
+  },
+  "devDependencies": {
+    "ava": "^5.3.1",
+    "dotenv": "^16.3.1",
+    "nock": "^13.3.0"
+  },
+  "ava": {
+    "files": [
+      "tests/**/*",
+      "!tests/test-files/**/*",
+      "!tests/test-docs/**/*",
+      "!tests/mocks/**/*"
+    ],
+    "timeout": "1m",
+    "nodeArguments": [
+      "--experimental-modules"
+    ],
+    "serial": true
   }
 }

package/helper-apps/cortex-file-handler/redis.js

@@ -96,34 +96,25 @@ const removeFromFileStoreMap = async (key) => {
 const cleanupRedisFileStoreMap = async (nDays=1) => {
     let cleaned = [];
     try {
-        // Get all key-value pairs from "FileStoreMap"
-        const fileStoreMap = await getAllFileStoreMap();
+        const map = await getAllFileStoreMap();
+        const nDaysAgo = new Date(Date.now() - nDays * 24 * 60 * 60 * 1000);
 
-        if(!fileStoreMap){
-            console.log("FileStoreMap is empty");
-            return;
-        }
-        
-        // Iterate over each key-value pair in the fileStoreMap
-        for (const [key, value] of Object.entries(fileStoreMap)) {
-            //check timestamp of each value compare to nDays and remove if older
-            const timestamp = new Date(value.timestamp);
-            const now = new Date();
-            const diffTime = Math.abs(now - timestamp);
-            const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
-            if (diffDays > nDays) {
+        for(const key in map){
+            const value = map[key];
+            const timestamp = value?.timestamp ? new Date(value.timestamp) : null;
+            if(!timestamp || timestamp.getTime() < nDaysAgo.getTime()){
                 // Remove the key from the "FileStoreMap" hash map
                 await removeFromFileStoreMap(key);
                 console.log(`Removed key ${key} from FileStoreMap`);
                 cleaned.push(Object.assign({hash:key}, value));
             }
-
         }
     } catch (error) {
         console.error(`Error cleaning FileStoreMap: ${error}`);
-    }finally{
-        return cleaned;
+    } finally {
+        // Cleanup code if needed
     }
+    return cleaned;
 };
 
 

package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js

@@ -0,0 +1,22 @@
+import { BlobServiceClient } from '@azure/storage-blob';
+
+async function createContainer() {
+    try {
+        const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
+        const containerClient = blobServiceClient.getContainerClient("test-container");
+        
+        console.log("Creating container...");
+        await containerClient.create();
+        console.log("Container created successfully");
+    } catch (error) {
+        // Ignore if container already exists
+        if (error.statusCode === 409) {
+            console.log("Container already exists");
+        } else {
+            console.error("Error creating container:", error);
+            process.exit(1);
+        }
+    }
+}
+
+createContainer(); 

package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js

@@ -0,0 +1,49 @@
+import { BlobServiceClient } from '@azure/storage-blob';
+import { Storage } from '@google-cloud/storage';
+
+async function createAzureContainer() {
+    try {
+        const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
+        const containerClient = blobServiceClient.getContainerClient("test-container");
+        
+        console.log("Creating Azure container...");
+        await containerClient.create();
+        console.log("Azure container created successfully");
+    } catch (error) {
+        // Ignore if container already exists
+        if (error.statusCode === 409) {
+            console.log("Azure container already exists");
+        } else {
+            console.error("Error creating Azure container:", error);
+            process.exit(1);
+        }
+    }
+}
+
+async function createGCSBucket() {
+    try {
+        const storage = new Storage({
+            projectId: "test-project",
+            apiEndpoint: "http://localhost:4443",
+        });
+        
+        console.log("Creating GCS bucket...");
+        await storage.createBucket("cortextempfiles");
+        console.log("GCS bucket created successfully");
+    } catch (error) {
+        // Ignore if bucket already exists
+        if (error.code === 409) {
+            console.log("GCS bucket already exists");
+        } else {
+            console.error("Error creating GCS bucket:", error);
+            process.exit(1);
+        }
+    }
+}
+
+async function setup() {
+    await createAzureContainer();
+    await createGCSBucket();
+}
+
+setup(); 

package/helper-apps/cortex-file-handler/scripts/test-azure.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Create temp directory for Azurite
+AZURITE_DIR="/tmp/azurite-test"
+mkdir -p $AZURITE_DIR
+
+# Start Azurite in background
+echo "Starting Azurite..."
+azurite --silent --location $AZURITE_DIR &
+AZURITE_PID=$!
+
+# Wait for Azurite to start
+sleep 2
+
+# Create test container
+echo "Setting up Azure container..."
+node scripts/setup-azure-container.js
+
+# Run the tests
+echo "Running tests..."
+node -r dotenv/config node_modules/ava/entrypoints/cli.mjs "$@"
+
+# Store test result
+TEST_RESULT=$?
+
+# Kill Azurite
+echo "Cleaning up..."
+kill $AZURITE_PID
+
+# Clean up Azurite directory
+rm -rf $AZURITE_DIR
+
+# Exit with test result
+exit $TEST_RESULT 

package/helper-apps/cortex-file-handler/scripts/test-gcs.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Exit on error
+set -e
+
+cleanup() {
+    echo "Cleaning up..."
+    if [ ! -z "$AZURITE_PID" ]; then
+        kill $AZURITE_PID 2>/dev/null || true
+    fi
+    docker stop fake-gcs-server 2>/dev/null || true
+    docker rm fake-gcs-server 2>/dev/null || true
+}
+
+# Set up cleanup trap
+trap cleanup EXIT
+
+echo "Starting test environment..."
+
+# Start Azurite if not running
+if ! nc -z localhost 10000; then
+    echo "Starting Azurite..."
+    azurite --silent --location .azurite --debug .azurite/debug.log &
+    AZURITE_PID=$!
+    # Wait for Azurite to be ready
+    until nc -z localhost 10000; do
+        sleep 1
+    done
+fi
+
+# Start fake-gcs-server if not running
+if ! nc -z localhost 4443; then
+    echo "Starting fake-gcs-server..."
+    docker run -d --name fake-gcs-server \
+        -p 4443:4443 \
+        fsouza/fake-gcs-server -scheme http
+    # Wait for fake-gcs-server to be ready
+    until nc -z localhost 4443; do
+        sleep 1
+    done
+fi
+
+# Create containers
+echo "Setting up test containers..."
+node scripts/setup-test-containers.js
+
+# Run the tests
+echo "Running tests..."
+node -r dotenv/config node_modules/ava/entrypoints/cli.mjs "$@" 

package/helper-apps/cortex-file-handler/start.js

@@ -1,8 +1,9 @@
-import MediaFileChunker from "./index.js";
+import CortexFileHandler from "./index.js";
 import express from "express";
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
 import cors from 'cors';
+import { readFileSync } from 'fs';
 
 import { publicIpv4 } from 'public-ip';
 const ipAddress = await publicIpv4();
@@ -11,18 +12,52 @@ const app = express();
 const port = process.env.PORT || 7071;
 const publicFolder = join(dirname(fileURLToPath(import.meta.url)), 'files');
 
+// Get version from package.json
+const packageJson = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'package.json'), 'utf8'));
+const version = packageJson.version;
+
 app.use(cors());
 // Serve static files from the public folder
 app.use('/files', express.static(publicFolder));
 
+// Health check endpoint
+app.get('/health', (req, res) => {
+    res.status(200).json({
+        status: 'healthy',
+        version: version
+    });
+});
+
+// New primary endpoint
+app.all('/api/CortexFileHandler', async (req, res) => {
+    const context = { req, res, log: console.log }
+    try {
+        await CortexFileHandler(context, req);
+        context.log(context.res);
+        res.status(context.res.status || 200).send(context.res.body);
+    } catch (error) {
+        const status = error.status || 500;
+        const message = error.message || 'Internal server error';
+        res.status(status).send(message);
+    }
+});
+
+// Legacy endpoint for compatibility
 app.all('/api/MediaFileChunker', async (req, res) => {
     const context = { req, res, log: console.log }
-    await MediaFileChunker(context, req);
-    res.send(context.res.body);
+    try {
+        await CortexFileHandler(context, req);
+        context.log(context.res);
+        res.status(context.res.status || 200).send(context.res.body);
+    } catch (error) {
+        const status = error.status || 500;
+        const message = error.message || 'Internal server error';
+        res.status(status).send(message);
+    }
 });
 
 app.listen(port, () => {
-    console.log(`MediaFileChunker helper running on port ${port}`);
+    console.log(`Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`);
 });
 
 export { port, publicFolder, ipAddress };

package/helper-apps/cortex-file-handler/tests/blobHandler.test.js

@@ -0,0 +1,292 @@
+import test from 'ava';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { uploadBlob, ensureGCSUpload, gcsUrlExists, deleteGCS, getBlobClient } from '../blobHandler.js';
+import axios from 'axios';
+import CortexFileHandler from '../index.js';
+import { setFileStoreMap } from '../redis.js';
+import { urlExists } from '../helper.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Helper function to determine if GCS is configured
+function isGCSConfigured() {
+    return process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY;
+}
+
+// Helper function to check file size in GCS
+async function getGCSFileSize(gcsUrl) {
+    if (!isGCSConfigured()) return null;
+    try {
+        const bucket = gcsUrl.split('/')[2];
+        const filename = gcsUrl.split('/').slice(3).join('/');
+
+        if (process.env.STORAGE_EMULATOR_HOST) {
+            const response = await axios.get(
+                `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${bucket}/o/${encodeURIComponent(filename)}`,
+                { validateStatus: status => status === 200 || status === 404 }
+            );
+            if (response.status === 200) {
+                return parseInt(response.data.size);
+            }
+            return null;
+        }
+    } catch (error) {
+        return null;
+    }
+}
+
+// Helper function to check file size in Azure/HTTP
+async function getHttpFileSize(url) {
+    try {
+        const response = await axios.head(url);
+        const contentLength = response.headers['content-length'];
+        return contentLength ? parseInt(contentLength) : null;
+    } catch (error) {
+        console.error('Error getting HTTP file size:', error);
+        return null;
+    }
+}
+
+test('test GCS backup during initial upload', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000); // Create a decent sized file
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // Upload the file - should go to both Azure/local and GCS
+        const context = { log: console.log };
+        const result = await uploadBlob(context, null, false, testFile);
+        
+        // Verify we got both URLs
+        t.truthy(result.url, 'Should have primary storage URL');
+        t.truthy(result.gcs, 'Should have GCS backup URL');
+        
+        // Verify GCS file exists
+        const gcsExists = await gcsUrlExists(result.gcs);
+        t.true(gcsExists, 'File should exist in GCS');
+        
+        // Verify file content size in GCS
+        const gcsSize = await getGCSFileSize(result.gcs);
+        t.is(gcsSize, testContent.length, 'GCS file size should match original');
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('test GCS backup restoration when missing', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000); // Create a decent sized file
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // First upload normally
+        const context = { log: console.log };
+        const result = await uploadBlob(context, null, false, testFile);
+        
+        // Verify initial upload worked
+        t.truthy(result.gcs, 'Should have GCS backup URL after initial upload');
+        
+        // Delete the GCS file
+        const gcsFileName = result.gcs.replace('gs://cortextempfiles/', '');
+        await deleteGCS(gcsFileName);
+        
+        // Verify file is gone
+        const existsAfterDelete = await gcsUrlExists(result.gcs);
+        t.false(existsAfterDelete, 'File should not exist in GCS after deletion');
+        
+        // Remove GCS URL to simulate missing backup
+        const { gcs: _, ...fileInfo } = result; // eslint-disable-line no-unused-vars
+        
+        // Try to ensure GCS backup
+        const updatedResult = await ensureGCSUpload(context, fileInfo);
+        
+        // Verify GCS URL was added
+        t.truthy(updatedResult.gcs, 'Should have GCS backup URL after ensure');
+        
+        // Verify GCS file exists
+        const gcsExists = await gcsUrlExists(updatedResult.gcs);
+        t.true(gcsExists, 'File should exist in GCS after ensure');
+        
+        // Verify file content size in GCS
+        const gcsSize = await getGCSFileSize(updatedResult.gcs);
+        t.is(gcsSize, testContent.length, 'GCS file size should match original after ensure');
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('test primary storage restoration from GCS backup', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000);
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // First upload normally
+        const context = { log: console.log };
+        const initialResult = await uploadBlob(context, null, false, testFile);
+        
+        // Verify initial upload worked
+        t.truthy(initialResult.url, 'Should have primary storage URL');
+        t.truthy(initialResult.gcs, 'Should have GCS backup URL');
+        
+        // Store the hash and simulate a missing primary file by requesting with a bad URL
+        const hash = 'test_primary_restore';
+        const modifiedResult = {
+            ...initialResult,
+            url: initialResult.url.replace('.blob.core.windows.net', '.invalid.url')
+        };
+        
+        // Set up Redis state with the bad URL
+        await setFileStoreMap(hash, modifiedResult);
+        
+        // Set up request for the handler
+        const mockReq = {
+            method: 'GET',
+            body: { params: { hash, checkHash: true } }
+        };
+        
+        // Set up context for the handler
+        const handlerContext = { 
+            log: console.log,
+            res: null
+        };
+        
+        // Call the handler which should restore from GCS
+        await CortexFileHandler(handlerContext, mockReq);
+        
+        // Verify we got a valid response
+        t.is(handlerContext.res.status, 200, 'Should get successful response');
+        t.truthy(handlerContext.res.body.url, 'Should have restored primary URL');
+        t.truthy(handlerContext.res.body.gcs, 'Should still have GCS URL');
+        
+        // Verify the restored URL is accessible
+        const { valid } = await urlExists(handlerContext.res.body.url);
+        t.true(valid, 'Restored URL should be accessible');
+
+        // Verify file sizes match in both storages
+        const gcsSize = await getGCSFileSize(handlerContext.res.body.gcs);
+        const azureSize = await getHttpFileSize(handlerContext.res.body.url);
+        t.is(azureSize, testContent.length, 'Azure file size should match original');
+        t.is(gcsSize, azureSize, 'Azure and GCS file sizes should match');
+        
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('test hash check returns 404 when both storages are empty', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000);
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // First upload normally
+        const context = { log: console.log };
+        const initialResult = await uploadBlob(context, null, false, testFile);
+        
+        // Verify initial upload worked
+        t.truthy(initialResult.url, 'Should have primary storage URL');
+        t.truthy(initialResult.gcs, 'Should have GCS backup URL');
+        
+        // Store the hash
+        const hash = 'test_both_missing';
+        await setFileStoreMap(hash, initialResult);
+        
+        // Verify both files exist initially
+        const initialPrimaryCheck = await urlExists(initialResult.url);
+        const initialGcsCheck = await gcsUrlExists(initialResult.gcs);
+        t.true(initialPrimaryCheck.valid, 'Primary file should exist initially');
+        t.true(initialGcsCheck, 'GCS file should exist initially');
+
+        // Delete from Azure/primary storage
+        const azureUrl = new URL(initialResult.url);
+        console.log('Azure URL:', initialResult.url);
+        // Get the path without query parameters and decode it
+        const fullPath = decodeURIComponent(azureUrl.pathname);
+        console.log('Full path:', fullPath);
+        // Get the request ID and filename from the path
+        const pathParts = fullPath.split('/');
+        const blobName = pathParts[pathParts.length - 1];
+        console.log('Attempting to delete Azure blob:', blobName);
+        
+        // Delete the blob using the correct container name
+        const { containerClient } = await getBlobClient();
+        const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+        await blockBlobClient.delete();
+        console.log('Azure deletion completed');
+
+        // Add a small delay to ensure deletion is complete
+        await new Promise(resolve => setTimeout(resolve, 1000));
+
+        // Delete from GCS
+        const gcsFileName = initialResult.gcs.replace('gs://cortextempfiles/', '');
+        console.log('Attempting to delete GCS file:', gcsFileName);
+        await deleteGCS(gcsFileName);
+        console.log('GCS deletion completed');
+        
+        // Verify both files are gone
+        const primaryExists = await urlExists(initialResult.url);
+        console.log('Primary exists after deletion:', primaryExists.valid);
+        const gcsExists = await gcsUrlExists(initialResult.gcs);
+        console.log('GCS exists after deletion:', gcsExists);
+        t.false(primaryExists.valid, 'Primary file should be deleted');
+        t.false(gcsExists, 'GCS file should be deleted');
+        
+        // Try to get the file via hash - should fail
+        const handlerContext = { 
+            log: console.log,
+            res: null
+        };
+        
+        await CortexFileHandler(handlerContext, {
+            method: 'GET',
+            body: { params: { hash, checkHash: true } }
+        });
+        
+        // Verify we got a 404 response
+        t.is(handlerContext.res.status, 404, 'Should get 404 when both files are missing');
+        t.true(handlerContext.res.body.includes('not found in storage'), 'Should indicate files are missing in storage');
+        
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});