@aj-archipelago/cortex 1.3.49 → 1.3.51

package/helper-apps/cortex-file-handler/start.js

@@ -1,63 +1,88 @@
 import CortexFileHandler from "./index.js";
 import express from "express";
-import { fileURLToPath } from 'url';
-import { dirname, join } from 'path';
-import cors from 'cors';
-import { readFileSync } from 'fs';
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import cors from "cors";
+import { readFileSync } from "fs";
 
-import { publicIpv4 } from 'public-ip';
-const ipAddress = await publicIpv4();
+import { publicIpv4 } from "public-ip";
+
+// When running under tests we want all generated URLs to resolve to the
+// locally-running server, otherwise checks like HEAD requests inside the
+// handler will fail (because the external IP is not reachable from inside
+// the test runner).  Use the machine's public IP in normal operation, but
+// fall back to "localhost" when the environment variable NODE_ENV indicates
+// a test run.
+
+let ipAddress = "localhost";
+if (process.env.NODE_ENV !== "test") {
+  try {
+    ipAddress = await publicIpv4();
+  } catch (err) {
+    // In rare cases querying the public IP can fail (e.g. no network when
+    // running offline).  Keep the default of "localhost" in that case so we
+    // still generate valid URLs.
+    console.warn("Unable to determine public IPv4 address – defaulting to 'localhost'.", err);
+  }
+}
 
 const app = express();
 const port = process.env.PORT || 7071;
-const publicFolder = join(dirname(fileURLToPath(import.meta.url)), 'files');
+const publicFolder = join(dirname(fileURLToPath(import.meta.url)), "files");
 
 // Get version from package.json
-const packageJson = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'package.json'), 'utf8'));
+const packageJson = JSON.parse(
+  readFileSync(
+    join(dirname(fileURLToPath(import.meta.url)), "package.json"),
+    "utf8",
+  ),
+);
 const version = packageJson.version;
 
 app.use(cors());
 // Serve static files from the public folder
-app.use('/files', express.static(publicFolder));
+app.use("/files", express.static(publicFolder));
 
 // Health check endpoint
-app.get('/health', (req, res) => {
-    res.status(200).json({
-        status: 'healthy',
-        version: version
-    });
+app.get("/health", (req, res) => {
+  res.status(200).json({
+    status: "healthy",
+    version: version,
+  });
 });
 
 // New primary endpoint
-app.all('/api/CortexFileHandler', async (req, res) => {
-    const context = { req, res, log: console.log }
-    try {
-        await CortexFileHandler(context, req);
-        context.log(context.res);
-        res.status(context.res.status || 200).send(context.res.body);
-    } catch (error) {
-        const status = error.status || 500;
-        const message = error.message || 'Internal server error';
-        res.status(status).send(message);
-    }
+app.all("/api/CortexFileHandler", async (req, res) => {
+  const context = { req, res, log: console.log };
+  try {
+    await CortexFileHandler(context, req);
+    context.log(context.res);
+    res.status(context.res.status || 200).send(context.res.body);
+  } catch (error) {
+    const status = error.status || 500;
+    const message = error.message || "Internal server error";
+    res.status(status).send(message);
+  }
 });
 
 // Legacy endpoint for compatibility
-app.all('/api/MediaFileChunker', async (req, res) => {
-    const context = { req, res, log: console.log }
-    try {
-        await CortexFileHandler(context, req);
-        context.log(context.res);
-        res.status(context.res.status || 200).send(context.res.body);
-    } catch (error) {
-        const status = error.status || 500;
-        const message = error.message || 'Internal server error';
-        res.status(status).send(message);
-    }
+app.all("/api/MediaFileChunker", async (req, res) => {
+  const context = { req, res, log: console.log };
+  try {
+    await CortexFileHandler(context, req);
+    context.log(context.res);
+    res.status(context.res.status || 200).send(context.res.body);
+  } catch (error) {
+    const status = error.status || 500;
+    const message = error.message || "Internal server error";
+    res.status(status).send(message);
+  }
 });
 
 app.listen(port, () => {
-    console.log(`Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`);
+  console.log(
+    `Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`,
+  );
 });
 
-export { port, publicFolder, ipAddress };
+export { port, publicFolder, ipAddress };

package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js

@@ -0,0 +1,144 @@
+import fs from 'fs/promises';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import test from 'ava';
+import axios from 'axios';
+import XLSX from 'xlsx';
+import { FileConversionService } from '../services/FileConversionService.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Mock context
+const mockContext = {
+    log: console.log
+};
+
+// Setup: Create test documents
+test.before(async (t) => {
+    const testDir = join(__dirname, 'test-docs');
+    await fs.mkdir(testDir, { recursive: true });
+
+    // Create various test files
+    const textFile = join(testDir, 'test.txt');
+    const largeTextFile = join(testDir, 'large.txt');
+    const unicodeFile = join(testDir, 'unicode.txt');
+    const jsonFile = join(testDir, 'test.json');
+    const emptyFile = join(testDir, 'empty.txt');
+    const excelFile = join(testDir, 'test.xlsx');
+
+    // Regular text content
+    await fs.writeFile(
+        textFile,
+        'This is a test document content.\nIt has multiple lines.\nThird line here.',
+    );
+
+    // Large text content (>100KB)
+    const largeContent = 'Lorem ipsum '.repeat(10000);
+    await fs.writeFile(largeTextFile, largeContent);
+
+    // Unicode content
+    const unicodeContent =
+    '这是中文内容\nこれは日本語です\nЭто русский текст\n🌟 emoji test';
+    await fs.writeFile(unicodeFile, unicodeContent);
+
+    // JSON content
+    await fs.writeFile(jsonFile, JSON.stringify({ test: 'content' }));
+
+    // Empty file
+    await fs.writeFile(emptyFile, '');
+
+    // Create a test Excel file
+    const workbook = XLSX.utils.book_new();
+    const ws1 = XLSX.utils.aoa_to_sheet([
+        ['Header 1', 'Header 2'],
+        ['Data 1', 'Data 2'],
+        ['Data 3', 'Data 4']
+    ]);
+    XLSX.utils.book_append_sheet(workbook, ws1, 'Sheet1');
+    XLSX.writeFile(workbook, excelFile);
+
+    t.context = {
+        testDir,
+        textFile,
+        largeTextFile,
+        unicodeFile,
+        jsonFile,
+        emptyFile,
+        excelFile
+    };
+});
+
+// Cleanup
+test.after.always(async (t) => {
+    await fs.rm(t.context.testDir, { recursive: true, force: true });
+});
+
+// Test Excel to CSV conversion
+test('converts Excel to CSV successfully', async (t) => {
+    const service = new FileConversionService(mockContext);
+    const result = await service.convertFile(t.context.excelFile);
+    
+    t.true(result.converted);
+    t.true(result.convertedPath.endsWith('.csv'));
+    
+    // Read the converted file and verify content
+    const content = await fs.readFile(result.convertedPath, 'utf-8');
+    t.true(content.includes('Header 1,Header 2'));
+    t.true(content.includes('Data 1,Data 2'));
+    t.true(content.includes('Data 3,Data 4'));
+});
+
+// Test document conversion with MarkItDown API
+test('converts document to markdown via MarkItDown API', async (t) => {
+    // Mock axios.get for MarkItDown API
+    const originalAxiosGet = axios.get;
+    axios.get = async (url) => {
+        if (url.includes('test.docx')) {
+            return {
+                data: {
+                    markdown: '# Test Document\n\nThis is a test document converted to markdown.'
+                }
+            };
+        }
+        throw new Error('Invalid URL');
+    };
+
+    const service = new FileConversionService(mockContext);
+    const result = await service.convertFile('test.docx', 'https://example.com/test.docx');
+    
+    t.true(result.converted);
+    t.true(result.convertedPath.endsWith('.md'));
+    
+    // Read the converted file and verify content
+    const content = await fs.readFile(result.convertedPath, 'utf-8');
+    t.true(content.includes('# Test Document'));
+    t.true(content.includes('This is a test document converted to markdown'));
+
+    // Restore original axios.get
+    axios.get = originalAxiosGet;
+});
+
+// Test error handling for missing original URL
+test('handles missing original URL for document conversion', async (t) => {
+    const service = new FileConversionService(mockContext);
+    await t.throwsAsync(
+        async () => service.convertFile('test.docx'),
+        { message: 'Original URL is required for document conversion' }
+    );
+});
+
+// Test error handling for unsupported file types
+test('handles unsupported file types', async (t) => {
+    const service = new FileConversionService(mockContext);
+    const result = await service.convertFile(t.context.jsonFile);
+    t.false(result.converted);
+});
+
+// Test file extension detection
+test('correctly detects file extensions', (t) => {
+    const service = new FileConversionService(mockContext);
+    t.true(service.needsConversion('test.docx'));
+    t.true(service.needsConversion('test.xlsx'));
+    t.false(service.needsConversion('test.txt'));
+    t.false(service.needsConversion('test.json'));
+}); 

package/helper-apps/cortex-file-handler/tests/blobHandler.test.js

@@ -1,19 +1,30 @@
-import test from 'ava';
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { uploadBlob, ensureGCSUpload, gcsUrlExists, deleteGCS, getBlobClient } from '../blobHandler.js';
+
+import test from 'ava';
 import axios from 'axios';
+
+import {
+    uploadBlob,
+    ensureGCSUpload,
+    gcsUrlExists,
+    deleteGCS,
+    getBlobClient,
+} from '../blobHandler.js';
+import { urlExists } from '../helper.js';
 import CortexFileHandler from '../index.js';
 import { setFileStoreMap } from '../redis.js';
-import { urlExists } from '../helper.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 // Helper function to determine if GCS is configured
 function isGCSConfigured() {
-    return process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY;
+    return (
+        process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 ||
+    process.env.GCP_SERVICE_ACCOUNT_KEY
+    );
 }
 
 // Helper function to check file size in GCS
@@ -26,7 +37,7 @@ async function getGCSFileSize(gcsUrl) {
         if (process.env.STORAGE_EMULATOR_HOST) {
             const response = await axios.get(
                 `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${bucket}/o/${encodeURIComponent(filename)}`,
-                { validateStatus: status => status === 200 || status === 404 }
+                { validateStatus: (status) => status === 200 || status === 404 },
             );
             if (response.status === 200) {
                 return parseInt(response.data.size);
@@ -50,7 +61,7 @@ async function getHttpFileSize(url) {
     }
 }
 
-test('test GCS backup during initial upload', async t => {
+test('test GCS backup during initial upload', async (t) => {
     if (!isGCSConfigured()) {
         t.pass('Skipping test - GCS not configured');
         return;
@@ -60,32 +71,32 @@ test('test GCS backup during initial upload', async t => {
     const testContent = 'Hello World!'.repeat(1000); // Create a decent sized file
     const testFile = path.join(__dirname, 'test.txt');
     fs.writeFileSync(testFile, testContent);
-    
+
     try {
-        // Upload the file - should go to both Azure/local and GCS
+    // Upload the file - should go to both Azure/local and GCS
         const context = { log: console.log };
         const result = await uploadBlob(context, null, false, testFile);
-        
+
         // Verify we got both URLs
         t.truthy(result.url, 'Should have primary storage URL');
         t.truthy(result.gcs, 'Should have GCS backup URL');
-        
+
         // Verify GCS file exists
         const gcsExists = await gcsUrlExists(result.gcs);
         t.true(gcsExists, 'File should exist in GCS');
-        
+
         // Verify file content size in GCS
         const gcsSize = await getGCSFileSize(result.gcs);
         t.is(gcsSize, testContent.length, 'GCS file size should match original');
     } finally {
-        // Cleanup
+    // Cleanup
         if (fs.existsSync(testFile)) {
             fs.unlinkSync(testFile);
         }
     }
 });
 
-test('test GCS backup restoration when missing', async t => {
+test('test GCS backup restoration when missing', async (t) => {
     if (!isGCSConfigured()) {
         t.pass('Skipping test - GCS not configured');
         return;
@@ -95,48 +106,52 @@ test('test GCS backup restoration when missing', async t => {
     const testContent = 'Hello World!'.repeat(1000); // Create a decent sized file
     const testFile = path.join(__dirname, 'test.txt');
     fs.writeFileSync(testFile, testContent);
-    
+
     try {
-        // First upload normally
+    // First upload normally
         const context = { log: console.log };
         const result = await uploadBlob(context, null, false, testFile);
-        
+
         // Verify initial upload worked
         t.truthy(result.gcs, 'Should have GCS backup URL after initial upload');
-        
+
         // Delete the GCS file
         const gcsFileName = result.gcs.replace('gs://cortextempfiles/', '');
         await deleteGCS(gcsFileName);
-        
+
         // Verify file is gone
         const existsAfterDelete = await gcsUrlExists(result.gcs);
         t.false(existsAfterDelete, 'File should not exist in GCS after deletion');
-        
+
         // Remove GCS URL to simulate missing backup
-        const { gcs: _, ...fileInfo } = result; // eslint-disable-line no-unused-vars
-        
+        const { gcs: _, ...fileInfo } = result;
+
         // Try to ensure GCS backup
         const updatedResult = await ensureGCSUpload(context, fileInfo);
-        
+
         // Verify GCS URL was added
         t.truthy(updatedResult.gcs, 'Should have GCS backup URL after ensure');
-        
+
         // Verify GCS file exists
         const gcsExists = await gcsUrlExists(updatedResult.gcs);
         t.true(gcsExists, 'File should exist in GCS after ensure');
-        
+
         // Verify file content size in GCS
         const gcsSize = await getGCSFileSize(updatedResult.gcs);
-        t.is(gcsSize, testContent.length, 'GCS file size should match original after ensure');
+        t.is(
+            gcsSize,
+            testContent.length,
+            'GCS file size should match original after ensure',
+        );
     } finally {
-        // Cleanup
+    // Cleanup
         if (fs.existsSync(testFile)) {
             fs.unlinkSync(testFile);
         }
     }
 });
 
-test('test primary storage restoration from GCS backup', async t => {
+test('test primary storage restoration from GCS backup', async (t) => {
     if (!isGCSConfigured()) {
         t.pass('Skipping test - GCS not configured');
         return;
@@ -146,46 +161,46 @@ test('test primary storage restoration from GCS backup', async t => {
     const testContent = 'Hello World!'.repeat(1000);
     const testFile = path.join(__dirname, 'test.txt');
     fs.writeFileSync(testFile, testContent);
-    
+
     try {
-        // First upload normally
+    // First upload normally
         const context = { log: console.log };
         const initialResult = await uploadBlob(context, null, false, testFile);
-        
+
         // Verify initial upload worked
         t.truthy(initialResult.url, 'Should have primary storage URL');
         t.truthy(initialResult.gcs, 'Should have GCS backup URL');
-        
+
         // Store the hash and simulate a missing primary file by requesting with a bad URL
         const hash = 'test_primary_restore';
         const modifiedResult = {
             ...initialResult,
-            url: initialResult.url.replace('.blob.core.windows.net', '.invalid.url')
+            url: initialResult.url.replace('test.txt', 'invalid.txt'),
         };
-        
+
         // Set up Redis state with the bad URL
         await setFileStoreMap(hash, modifiedResult);
-        
+
         // Set up request for the handler
         const mockReq = {
             method: 'GET',
-            body: { params: { hash, checkHash: true } }
+            body: { params: { hash, checkHash: true } },
         };
-        
+
         // Set up context for the handler
-        const handlerContext = { 
+        const handlerContext = {
             log: console.log,
-            res: null
+            res: null,
         };
-        
+
         // Call the handler which should restore from GCS
         await CortexFileHandler(handlerContext, mockReq);
-        
+
         // Verify we got a valid response
         t.is(handlerContext.res.status, 200, 'Should get successful response');
         t.truthy(handlerContext.res.body.url, 'Should have restored primary URL');
         t.truthy(handlerContext.res.body.gcs, 'Should still have GCS URL');
-        
+
         // Verify the restored URL is accessible
         const { valid } = await urlExists(handlerContext.res.body.url);
         t.true(valid, 'Restored URL should be accessible');
@@ -193,18 +208,21 @@ test('test primary storage restoration from GCS backup', async t => {
         // Verify file sizes match in both storages
         const gcsSize = await getGCSFileSize(handlerContext.res.body.gcs);
         const azureSize = await getHttpFileSize(handlerContext.res.body.url);
-        t.is(azureSize, testContent.length, 'Azure file size should match original');
+        t.is(
+            azureSize,
+            testContent.length,
+            'Azure file size should match original',
+        );
         t.is(gcsSize, azureSize, 'Azure and GCS file sizes should match');
-        
     } finally {
-        // Cleanup
+    // Cleanup
         if (fs.existsSync(testFile)) {
             fs.unlinkSync(testFile);
         }
     }
 });
 
-test('test hash check returns 404 when both storages are empty', async t => {
+test('test hash check returns 404 when both storages are empty', async (t) => {
     if (!isGCSConfigured()) {
         t.pass('Skipping test - GCS not configured');
         return;
@@ -214,20 +232,20 @@ test('test hash check returns 404 when both storages are empty', async t => {
     const testContent = 'Hello World!'.repeat(1000);
     const testFile = path.join(__dirname, 'test.txt');
     fs.writeFileSync(testFile, testContent);
-    
+
     try {
-        // First upload normally
+    // First upload normally
         const context = { log: console.log };
         const initialResult = await uploadBlob(context, null, false, testFile);
-        
+
         // Verify initial upload worked
         t.truthy(initialResult.url, 'Should have primary storage URL');
         t.truthy(initialResult.gcs, 'Should have GCS backup URL');
-        
+
         // Store the hash
         const hash = 'test_both_missing';
         await setFileStoreMap(hash, initialResult);
-        
+
         // Verify both files exist initially
         const initialPrimaryCheck = await urlExists(initialResult.url);
         const initialGcsCheck = await gcsUrlExists(initialResult.gcs);
@@ -244,7 +262,7 @@ test('test hash check returns 404 when both storages are empty', async t => {
         const pathParts = fullPath.split('/');
         const blobName = pathParts[pathParts.length - 1];
         console.log('Attempting to delete Azure blob:', blobName);
-        
+
         // Delete the blob using the correct container name
         const { containerClient } = await getBlobClient();
         const blockBlobClient = containerClient.getBlockBlobClient(blobName);
@@ -252,14 +270,14 @@ test('test hash check returns 404 when both storages are empty', async t => {
         console.log('Azure deletion completed');
 
         // Add a small delay to ensure deletion is complete
-        await new Promise(resolve => setTimeout(resolve, 1000));
+        await new Promise((resolve) => setTimeout(resolve, 1000));
 
         // Delete from GCS
         const gcsFileName = initialResult.gcs.replace('gs://cortextempfiles/', '');
         console.log('Attempting to delete GCS file:', gcsFileName);
         await deleteGCS(gcsFileName);
         console.log('GCS deletion completed');
-        
+
         // Verify both files are gone
         const primaryExists = await urlExists(initialResult.url);
         console.log('Primary exists after deletion:', primaryExists.valid);
@@ -267,26 +285,32 @@ test('test hash check returns 404 when both storages are empty', async t => {
         console.log('GCS exists after deletion:', gcsExists);
         t.false(primaryExists.valid, 'Primary file should be deleted');
         t.false(gcsExists, 'GCS file should be deleted');
-        
+
         // Try to get the file via hash - should fail
-        const handlerContext = { 
+        const handlerContext = {
             log: console.log,
-            res: null
+            res: null,
         };
-        
+
         await CortexFileHandler(handlerContext, {
             method: 'GET',
-            body: { params: { hash, checkHash: true } }
+            body: { params: { hash, checkHash: true } },
         });
-        
+
         // Verify we got a 404 response
-        t.is(handlerContext.res.status, 404, 'Should get 404 when both files are missing');
-        t.true(handlerContext.res.body.includes('not found in storage'), 'Should indicate files are missing in storage');
-        
+        t.is(
+            handlerContext.res.status,
+            404,
+            'Should get 404 when both files are missing',
+        );
+        t.true(
+            handlerContext.res.body.includes('not found in storage'),
+            'Should indicate files are missing in storage',
+        );
     } finally {
-        // Cleanup
+    // Cleanup
         if (fs.existsSync(testFile)) {
             fs.unlinkSync(testFile);
         }
     }
-}); 
+});