@aj-archipelago/cortex 1.3.10 → 1.3.12

package/helper-apps/cortex-file-handler/package.json

@@ -1,12 +1,15 @@
 {
-  "name": "@aj-archipelago/mediafilechunker",
-  "version": "1.0.0",
-  "description": "",
+  "name": "@aj-archipelago/cortex-file-handler",
+  "version": "1.0.15",
+  "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "scripts": {
     "start": "node start.js",
     "dev": "node -r dotenv/config start.js",
-    "test": "echo \"No tests yet...\""
+    "test": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs",
+    "test:azure": "DOTENV_CONFIG_PATH=.env.test.azure NODE_ENV=test ./scripts/test-azure.sh",
+    "test:watch": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs --watch",
+    "test:gcs": "DOTENV_CONFIG_PATH=.env.test.gcs NODE_ENV=test ./scripts/test-gcs.sh"
   },
   "dependencies": {
     "@azure/storage-blob": "^12.13.0",
@@ -24,5 +27,22 @@
     "public-ip": "^6.0.1",
     "uuid": "^9.0.0",
     "xlsx": "^0.18.5"
+  },
+  "devDependencies": {
+    "ava": "^5.3.1",
+    "dotenv": "^16.3.1",
+    "nock": "^13.3.0"
+  },
+  "ava": {
+    "files": [
+      "tests/**/*",
+      "!tests/test-files/**/*",
+      "!tests/test-docs/**/*",
+      "!tests/mocks/**/*"
+    ],
+    "timeout": "1m",
+    "nodeArguments": [
+      "--experimental-modules"
+    ]
   }
 }

package/helper-apps/cortex-file-handler/redis.js

@@ -96,34 +96,25 @@ const removeFromFileStoreMap = async (key) => {
 const cleanupRedisFileStoreMap = async (nDays=1) => {
     let cleaned = [];
     try {
-        // Get all key-value pairs from "FileStoreMap"
-        const fileStoreMap = await getAllFileStoreMap();
+        const map = await getAllFileStoreMap();
+        const nDaysAgo = new Date(Date.now() - nDays * 24 * 60 * 60 * 1000);
 
-        if(!fileStoreMap){
-            console.log("FileStoreMap is empty");
-            return;
-        }
-        
-        // Iterate over each key-value pair in the fileStoreMap
-        for (const [key, value] of Object.entries(fileStoreMap)) {
-            //check timestamp of each value compare to nDays and remove if older
-            const timestamp = new Date(value.timestamp);
-            const now = new Date();
-            const diffTime = Math.abs(now - timestamp);
-            const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
-            if (diffDays > nDays) {
+        for(const key in map){
+            const value = map[key];
+            const timestamp = value?.timestamp ? new Date(value.timestamp) : null;
+            if(!timestamp || timestamp.getTime() < nDaysAgo.getTime()){
                 // Remove the key from the "FileStoreMap" hash map
                 await removeFromFileStoreMap(key);
                 console.log(`Removed key ${key} from FileStoreMap`);
                 cleaned.push(Object.assign({hash:key}, value));
             }
-
         }
     } catch (error) {
         console.error(`Error cleaning FileStoreMap: ${error}`);
-    }finally{
-        return cleaned;
+    } finally {
+        // Cleanup code if needed
     }
+    return cleaned;
 };
 
 

package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js

@@ -0,0 +1,22 @@
+import { BlobServiceClient } from '@azure/storage-blob';
+
+async function createContainer() {
+    try {
+        const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
+        const containerClient = blobServiceClient.getContainerClient("test-container");
+        
+        console.log("Creating container...");
+        await containerClient.create();
+        console.log("Container created successfully");
+    } catch (error) {
+        // Ignore if container already exists
+        if (error.statusCode === 409) {
+            console.log("Container already exists");
+        } else {
+            console.error("Error creating container:", error);
+            process.exit(1);
+        }
+    }
+}
+
+createContainer(); 

package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js

@@ -0,0 +1,49 @@
+import { BlobServiceClient } from '@azure/storage-blob';
+import { Storage } from '@google-cloud/storage';
+
+async function createAzureContainer() {
+    try {
+        const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
+        const containerClient = blobServiceClient.getContainerClient("test-container");
+        
+        console.log("Creating Azure container...");
+        await containerClient.create();
+        console.log("Azure container created successfully");
+    } catch (error) {
+        // Ignore if container already exists
+        if (error.statusCode === 409) {
+            console.log("Azure container already exists");
+        } else {
+            console.error("Error creating Azure container:", error);
+            process.exit(1);
+        }
+    }
+}
+
+async function createGCSBucket() {
+    try {
+        const storage = new Storage({
+            projectId: "test-project",
+            apiEndpoint: "http://localhost:4443",
+        });
+        
+        console.log("Creating GCS bucket...");
+        await storage.createBucket("cortextempfiles");
+        console.log("GCS bucket created successfully");
+    } catch (error) {
+        // Ignore if bucket already exists
+        if (error.code === 409) {
+            console.log("GCS bucket already exists");
+        } else {
+            console.error("Error creating GCS bucket:", error);
+            process.exit(1);
+        }
+    }
+}
+
+async function setup() {
+    await createAzureContainer();
+    await createGCSBucket();
+}
+
+setup(); 

package/helper-apps/cortex-file-handler/scripts/test-azure.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Create temp directory for Azurite
+AZURITE_DIR="/tmp/azurite-test"
+mkdir -p $AZURITE_DIR
+
+# Start Azurite in background
+echo "Starting Azurite..."
+azurite --silent --location $AZURITE_DIR &
+AZURITE_PID=$!
+
+# Wait for Azurite to start
+sleep 2
+
+# Create test container
+echo "Setting up Azure container..."
+node scripts/setup-azure-container.js
+
+# Run the tests
+echo "Running tests..."
+node -r dotenv/config node_modules/ava/entrypoints/cli.mjs
+
+# Store test result
+TEST_RESULT=$?
+
+# Kill Azurite
+echo "Cleaning up..."
+kill $AZURITE_PID
+
+# Clean up Azurite directory
+rm -rf $AZURITE_DIR
+
+# Exit with test result
+exit $TEST_RESULT 

package/helper-apps/cortex-file-handler/scripts/test-gcs.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Exit on error
+set -e
+
+cleanup() {
+    echo "Cleaning up..."
+    if [ ! -z "$AZURITE_PID" ]; then
+        kill $AZURITE_PID 2>/dev/null || true
+    fi
+    docker stop fake-gcs-server 2>/dev/null || true
+    docker rm fake-gcs-server 2>/dev/null || true
+}
+
+# Set up cleanup trap
+trap cleanup EXIT
+
+echo "Starting test environment..."
+
+# Start Azurite if not running
+if ! nc -z localhost 10000; then
+    echo "Starting Azurite..."
+    azurite --silent --location .azurite --debug .azurite/debug.log &
+    AZURITE_PID=$!
+    # Wait for Azurite to be ready
+    until nc -z localhost 10000; do
+        sleep 1
+    done
+fi
+
+# Start fake-gcs-server if not running
+if ! nc -z localhost 4443; then
+    echo "Starting fake-gcs-server..."
+    docker run -d --name fake-gcs-server \
+        -p 4443:4443 \
+        fsouza/fake-gcs-server -scheme http
+    # Wait for fake-gcs-server to be ready
+    until nc -z localhost 4443; do
+        sleep 1
+    done
+fi
+
+# Create containers
+echo "Setting up test containers..."
+node scripts/setup-test-containers.js
+
+# Run the tests
+echo "Running tests..."
+node -r dotenv/config node_modules/ava/entrypoints/cli.mjs "$@" 

package/helper-apps/cortex-file-handler/start.js

@@ -1,4 +1,4 @@
-import MediaFileChunker from "./index.js";
+import CortexFileHandler from "./index.js";
 import express from "express";
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
@@ -15,14 +15,36 @@ app.use(cors());
 // Serve static files from the public folder
 app.use('/files', express.static(publicFolder));
 
+// New primary endpoint
+app.all('/api/CortexFileHandler', async (req, res) => {
+    const context = { req, res, log: console.log }
+    try {
+        await CortexFileHandler(context, req);
+        context.log(context.res);
+        res.status(context.res.status || 200).send(context.res.body);
+    } catch (error) {
+        const status = error.status || 500;
+        const message = error.message || 'Internal server error';
+        res.status(status).send(message);
+    }
+});
+
+// Legacy endpoint for compatibility
 app.all('/api/MediaFileChunker', async (req, res) => {
     const context = { req, res, log: console.log }
-    await MediaFileChunker(context, req);
-    res.send(context.res.body);
+    try {
+        await CortexFileHandler(context, req);
+        context.log(context.res);
+        res.status(context.res.status || 200).send(context.res.body);
+    } catch (error) {
+        const status = error.status || 500;
+        const message = error.message || 'Internal server error';
+        res.status(status).send(message);
+    }
 });
 
 app.listen(port, () => {
-    console.log(`MediaFileChunker helper running on port ${port}`);
+    console.log(`Cortex File Handler running on port ${port} (includes legacy MediaFileChunker endpoint)`);
 });
 
 export { port, publicFolder, ipAddress };

package/helper-apps/cortex-file-handler/tests/docHelper.test.js

@@ -0,0 +1,148 @@
+import test from 'ava';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import fs from 'fs/promises';
+import { documentToText, easyChunker } from '../docHelper.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Setup: Create test documents
+test.before(async t => {
+    const testDir = join(__dirname, 'test-docs');
+    await fs.mkdir(testDir, { recursive: true });
+    
+    // Create various test files
+    const textFile = join(testDir, 'test.txt');
+    const largeTextFile = join(testDir, 'large.txt');
+    const unicodeFile = join(testDir, 'unicode.txt');
+    const jsonFile = join(testDir, 'test.json');
+    const emptyFile = join(testDir, 'empty.txt');
+    
+    // Regular text content
+    await fs.writeFile(textFile, 'This is a test document content.\nIt has multiple lines.\nThird line here.');
+    
+    // Large text content (>100KB)
+    const largeContent = 'Lorem ipsum '.repeat(10000);
+    await fs.writeFile(largeTextFile, largeContent);
+    
+    // Unicode content
+    const unicodeContent = '这是中文内容\nこれは日本語です\nЭто русский текст\n🌟 emoji test';
+    await fs.writeFile(unicodeFile, unicodeContent);
+    
+    // JSON content
+    await fs.writeFile(jsonFile, JSON.stringify({ test: 'content' }));
+    
+    // Empty file
+    await fs.writeFile(emptyFile, '');
+
+    t.context = {
+        testDir,
+        textFile,
+        largeTextFile,
+        unicodeFile,
+        jsonFile,
+        emptyFile
+    };
+});
+
+// Cleanup
+test.after.always(async t => {
+    await fs.rm(t.context.testDir, { recursive: true, force: true });
+});
+
+// Test basic text file processing
+test('processes text files correctly', async t => {
+    const result = await documentToText(t.context.textFile, 'text/plain');
+    t.true(typeof result === 'string', 'Result should be a string');
+    t.true(result.includes('test document content'), 'Result should contain file content');
+    t.true(result.includes('multiple lines'), 'Result should preserve multiple lines');
+});
+
+// Test large file handling
+test('handles large text files', async t => {
+    const result = await documentToText(t.context.largeTextFile, 'text/plain');
+    t.true(result.length > 50000, 'Should handle large files');
+    t.true(result.includes('Lorem ipsum'), 'Should contain expected content');
+});
+
+// Test Unicode handling
+test('handles Unicode content correctly', async t => {
+    const result = await documentToText(t.context.unicodeFile, 'text/plain');
+    t.true(result.includes('这是中文内容'), 'Should preserve Chinese characters');
+    t.true(result.includes('これは日本語です'), 'Should preserve Japanese characters');
+    t.true(result.includes('Это русский текст'), 'Should preserve Cyrillic characters');
+    t.true(result.includes('🌟'), 'Should preserve emoji');
+});
+
+// Test JSON file handling
+test('rejects JSON files appropriately', async t => {
+    await t.throwsAsync(
+        async () => documentToText(t.context.jsonFile, 'application/json'),
+        { message: 'Unsupported file type: json' }
+    );
+});
+
+// Test empty file handling
+test('handles empty files appropriately', async t => {
+    const result = await documentToText(t.context.emptyFile, 'text/plain');
+    t.is(result, '', 'Empty file should return empty string');
+});
+
+// Test unsupported file types
+test('rejects unsupported file types', async t => {
+    const unsupportedFile = join(t.context.testDir, 'unsupported.xyz');
+    await fs.writeFile(unsupportedFile, 'test content');
+    await t.throwsAsync(
+        async () => documentToText(unsupportedFile, 'unsupported/type'),
+        { message: 'Unsupported file type: xyz' }
+    );
+});
+
+// Test text chunking functionality
+test('chunks text correctly with default settings', t => {
+    const text = 'This is a test.\nSecond line.\nThird line.\nFourth line.';
+    const chunks = easyChunker(text);
+    
+    t.true(Array.isArray(chunks), 'Should return an array of chunks');
+    t.true(chunks.length > 0, 'Should create at least one chunk');
+    t.true(chunks.every(chunk => typeof chunk === 'string'), 'All chunks should be strings');
+});
+
+// Test chunking with very long text
+test('handles chunking of long text', t => {
+    const longText = 'Test sentence. '.repeat(1000);
+    const chunks = easyChunker(longText);
+    
+    t.true(chunks.length > 1, 'Should split long text into multiple chunks');
+    t.true(chunks.every(chunk => chunk.length <= 10000), 'Each chunk should not exceed max length');
+});
+
+// Test chunking with various delimiters
+test('respects sentence boundaries in chunking', t => {
+    const text = 'First sentence. Second sentence! Third sentence? Fourth sentence.';
+    const chunks = easyChunker(text);
+    
+    t.true(chunks.every(chunk => 
+        chunk.match(/[.!?](\s|$)/) || chunk === chunks[chunks.length - 1]
+    ), 'Chunks should end with sentence delimiters when possible');
+});
+
+// Test chunking with newlines
+test('handles newlines in chunking', t => {
+    const text = 'Line 1\nLine 2\nLine 3\nLine 4';
+    const chunks = easyChunker(text);
+    
+    t.true(chunks.some(chunk => chunk.includes('\n')), 'Should preserve newlines');
+});
+
+// Test chunking edge cases
+test('handles chunking edge cases', t => {
+    // Empty string
+    t.deepEqual(easyChunker(''), [''], 'Should handle empty string');
+    
+    // Single character
+    t.deepEqual(easyChunker('a'), ['a'], 'Should handle single character');
+    
+    // Only whitespace
+    t.deepEqual(easyChunker('   \n   '), ['   \n   '], 'Should handle whitespace');
+});