@aj-archipelago/cortex 1.3.50 → 1.3.52

package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js

@@ -0,0 +1,53 @@
+/**
+ * Base interface for storage providers
+ */
+export class StorageProvider {
+    /**
+     * Upload a file to storage
+     * @param {Object} context - The context object
+     * @param {string} filePath - Path to the file to upload
+     * @param {string} requestId - Unique identifier for the request
+     * @param {string} [hash] - Optional hash of the file
+     * @returns {Promise<{url: string, blobName: string}>} The URL and blob name of the uploaded file
+     */
+    async uploadFile(context, filePath, requestId, hash = null) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Delete files associated with a request ID
+     * @param {string} requestId - The request ID to delete files for
+     * @returns {Promise<string[]>} Array of deleted file URLs
+     */
+    async deleteFiles(requestId) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Check if a file exists at the given URL
+     * @param {string} url - The URL to check
+     * @returns {Promise<boolean>} Whether the file exists
+     */
+    async fileExists(url) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Download a file from storage
+     * @param {string} url - The URL of the file to download
+     * @param {string} destinationPath - Where to save the downloaded file
+     * @returns {Promise<void>}
+     */
+    async downloadFile(url, destinationPath) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Clean up files by their URLs
+     * @param {string[]} urls - Array of URLs to clean up
+     * @returns {Promise<void>}
+     */
+    async cleanup(urls) {
+        throw new Error('Method not implemented');
+    }
+} 

package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js

@@ -0,0 +1,259 @@
+import { StorageFactory } from './StorageFactory.js';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+
+export class StorageService {
+    constructor(factory) {
+        this.factory = factory || new StorageFactory();
+        this.primaryProvider = this.factory.getPrimaryProvider();
+        this.backupProvider = this.factory.getGCSProvider();
+    }
+
+    getPrimaryProvider() {
+        return this.primaryProvider;
+    }
+
+    getBackupProvider() {
+        return this.backupProvider;
+    }
+
+    async uploadFile(...args) {
+        /*
+            Supported call shapes:
+            1) uploadFile(buffer, filename)
+            2) uploadFile(context, filePath, requestId, hash?) – legacy internal use
+        */
+
+        // Shape (buffer, filename)
+        if (args.length === 2 && Buffer.isBuffer(args[0]) && typeof args[1] === 'string') {
+            const buffer = args[0];
+            const filename = args[1];
+            const tempFile = path.join(os.tmpdir(), `${Date.now()}_${filename}`);
+            await fs.promises.writeFile(tempFile, buffer);
+            try {
+                const { url } = await this.primaryProvider.uploadFile({}, tempFile, filename);
+                return { url };
+            } finally {
+                if (fs.existsSync(tempFile)) {
+                    await fs.promises.unlink(tempFile).catch(() => {});
+                }
+            }
+        }
+
+        // Fallback to legacy (context, filePath, requestId, hash?)
+        const [context, filePath, requestId, hash] = args;
+        return this.uploadFileWithProviders(context, filePath, requestId, hash);
+    }
+
+    async uploadFileToBackup(fileOrBuffer, filename) {
+        if (!this.backupProvider) {
+            throw new Error('Backup provider not configured');
+        }
+
+        if (Buffer.isBuffer(fileOrBuffer)) {
+            const tempFile = path.join(os.tmpdir(), `${Date.now()}_${filename}`);
+            await fs.promises.writeFile(tempFile, fileOrBuffer);
+            try {
+                const result = await this.backupProvider.uploadFile({}, tempFile, filename);
+                return { url: result.url };
+            } finally {
+                if (fs.existsSync(tempFile)) {
+                    await fs.promises.unlink(tempFile).catch(() => {});
+                }
+            }
+        }
+
+        const result = await this.backupProvider.uploadFile({}, fileOrBuffer, filename);
+        return { url: result.url };
+    }
+
+    async downloadFile(url, destinationPath = null) {
+        const useBackup = url.startsWith('gs://');
+
+        if (useBackup && !this.backupProvider) {
+            throw new Error('Backup provider not configured');
+        }
+
+        // If caller supplied a destination path, stream to disk and return void
+        if (destinationPath) {
+            if (useBackup) {
+                await this.backupProvider.downloadFile(url, destinationPath);
+            } else {
+                await this.primaryProvider.downloadFile(url, destinationPath);
+            }
+            return;
+        }
+
+        // Otherwise download to a temp file and return Buffer
+        const tempFile = path.join(os.tmpdir(), path.basename(url));
+        try {
+            if (useBackup) {
+                await this.backupProvider.downloadFile(url, tempFile);
+            } else {
+                await this.primaryProvider.downloadFile(url, tempFile);
+            }
+            return await fs.promises.readFile(tempFile);
+        } finally {
+            if (fs.existsSync(tempFile)) {
+                await fs.promises.unlink(tempFile).catch(() => {});
+            }
+        }
+    }
+
+    async deleteFile(url) {
+        if (typeof this.primaryProvider.deleteFile === 'function') {
+            return await this.primaryProvider.deleteFile(url);
+        }
+        // Fallback for providers that only have deleteFiles
+        return await this.primaryProvider.deleteFiles([url]);
+    }
+
+    async deleteFileFromBackup(url) {
+        if (!this.backupProvider) {
+            throw new Error('Backup provider not configured');
+        }
+        if (typeof this.backupProvider.deleteFile === 'function') {
+            return await this.backupProvider.deleteFile(url);
+        }
+        // Fallback for providers that only have deleteFiles
+        return await this.backupProvider.deleteFiles([url]);
+    }
+
+    async uploadFileWithProviders(context, filePath, requestId, hash = null) {
+        const primaryResult = await this.primaryProvider.uploadFile(context, filePath, requestId, hash);
+
+        let gcsResult = null;
+        if (this.backupProvider) {
+            gcsResult = await this.backupProvider.uploadFile(context, filePath, requestId, hash);
+        }
+
+        return { ...primaryResult, gcs: gcsResult?.url };
+    }
+
+    async deleteFiles(requestId) {
+        if (!requestId) {
+            throw new Error('Missing requestId parameter');
+        }
+
+        const results = [];
+        
+        // Delete from primary storage
+        try {
+            const primaryResult = await this.primaryProvider.deleteFiles(requestId);
+            if (primaryResult && primaryResult.length > 0) {
+                results.push(...primaryResult);
+            }
+        } catch (error) {
+            console.error(`Error deleting files from primary storage for ${requestId}:`, error);
+        }
+
+        // If GCS is configured, delete from there too
+        if (this.backupProvider) {
+            try {
+                const gcsResult = await this.backupProvider.deleteFiles(requestId);
+                if (gcsResult && gcsResult.length > 0) {
+                    results.push(...gcsResult);
+                }
+            } catch (error) {
+                console.error(`Error deleting files from GCS for ${requestId}:`, error);
+            }
+        }
+
+        return results;
+    }
+
+    async fileExists(url) {
+        if (!url) {
+            return false;
+        }
+
+        try {
+            if (url.startsWith('gs://')) {
+                return this.backupProvider ? await this.backupProvider.fileExists(url) : false;
+            }
+            return await this.primaryProvider.fileExists(url);
+        } catch (error) {
+            console.error(`Error checking file existence for ${url}:`, error);
+            return false;
+        }
+    }
+
+    async cleanup(urls) {
+        if (!urls || !urls.length) return;
+
+        const results = [];
+        
+        // Split URLs by type
+        const primaryUrls = [];
+        const gcsUrls = [];
+        
+        for (const url of urls) {
+            if (url.startsWith('gs://')) {
+                gcsUrls.push(url);
+            } else {
+                primaryUrls.push(url);
+            }
+        }
+
+        // Clean up primary storage
+        if (primaryUrls.length > 0) {
+            const primaryResult = await this.primaryProvider.cleanup(primaryUrls);
+            results.push(...primaryResult);
+        }
+
+        // Clean up GCS if configured
+        if (gcsUrls.length > 0 && this.backupProvider) {
+            const gcsResult = await this.backupProvider.cleanup(gcsUrls);
+            results.push(...gcsResult);
+        }
+
+        return results;
+    }
+
+    async ensureGCSUpload(context, existingFile) {
+        if (!this.backupProvider || !existingFile.url || !this.backupProvider.isConfigured()) {
+            return existingFile;
+        }
+
+        // If we already have a GCS URL, check if it exists
+        if (existingFile.gcs) {
+            const exists = await this.backupProvider.fileExists(existingFile.gcs);
+            if (exists) {
+                return existingFile;
+            }
+        }
+
+        // Download from primary storage
+        const tempFile = path.join(os.tmpdir(), path.basename(existingFile.url));
+        try {
+            await this.primaryProvider.downloadFile(existingFile.url, tempFile);
+            
+            // Upload to GCS
+            const requestId = path.dirname(existingFile.blobName) || 'restore';
+            const gcsResult = await this.backupProvider.uploadFile(
+                context,
+                tempFile,
+                requestId
+            );
+
+            return {
+                ...existingFile,
+                gcs: gcsResult.url
+            };
+        } finally {
+            // Cleanup temp file
+            if (fs.existsSync(tempFile)) {
+                fs.unlinkSync(tempFile);
+            }
+        }
+    }
+
+    async downloadFileFromBackup(url, destinationPath = null) {
+        if (!this.backupProvider) {
+            throw new Error('Backup provider not configured');
+        }
+        // Delegate to the unified downloadFile handler
+        return await this.downloadFile(url, destinationPath);
+    }
+} 

package/helper-apps/cortex-file-handler/src/start.js

@@ -0,0 +1,88 @@
+import CortexFileHandler from "./index.js";
+import express from "express";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import cors from "cors";
+import { readFileSync } from "fs";
+
+import { publicIpv4 } from "public-ip";
+
+// When running under tests we want all generated URLs to resolve to the
+// locally-running server, otherwise checks like HEAD requests inside the
+// handler will fail (because the external IP is not reachable from inside
+// the test runner).  Use the machine's public IP in normal operation, but
+// fall back to "localhost" when the environment variable NODE_ENV indicates
+// a test run.
+
+let ipAddress = "localhost";
+if (process.env.NODE_ENV !== "test") {
+  try {
+    ipAddress = await publicIpv4();
+  } catch (err) {
+    // In rare cases querying the public IP can fail (e.g. no network when
+    // running offline).  Keep the default of "localhost" in that case so we
+    // still generate valid URLs.
+    console.warn("Unable to determine public IPv4 address – defaulting to 'localhost'.", err);
+  }
+}
+
+const app = express();
+const port = process.env.PORT || 7071;
+const publicFolder = join(dirname(fileURLToPath(import.meta.url)), "files");
+
+// Get version from package.json
+const packageJson = JSON.parse(
+  readFileSync(
+    join(dirname(fileURLToPath(import.meta.url)), "../package.json"),
+    "utf8",
+  ),
+);
+const version = packageJson.version;
+
+app.use(cors());
+// Serve static files from the public folder
+app.use("/files", express.static(publicFolder));
+
+// Health check endpoint
+app.get("/health", (req, res) => {
+  res.status(200).json({
+    status: "healthy",
+    version: version,
+  });
+});
+
+// New primary endpoint
+app.all("/api/CortexFileHandler", async (req, res) => {
+  const context = { req, res, log: console.log };
+  try {
+    await CortexFileHandler(context, req);
+    context.log(context.res);
+    res.status(context.res.status || 200).send(context.res.body);
+  } catch (error) {
+    const status = error.status || 500;
+    const message = error.message || "Internal server error";
+    res.status(status).send(message);
+  }
+});
+
+// Legacy endpoint for compatibility
+app.all("/api/MediaFileChunker", async (req, res) => {
+  const context = { req, res, log: console.log };
+  try {
+    await CortexFileHandler(context, req);
+    context.log(context.res);
+    res.status(context.res.status || 200).send(context.res.body);
+  } catch (error) {
+    const status = error.status || 500;
+    const message = error.message || "Internal server error";
+    res.status(status).send(message);
+  }
+});
+
+app.listen(port, () => {
+  console.log(
+    `Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`,
+  );
+});
+
+export { port, publicFolder, ipAddress };

package/helper-apps/cortex-file-handler/src/utils/filenameUtils.js

@@ -0,0 +1,28 @@
+import path from 'path';
+
+/**
+ * Sanitize a filename so that it is safe and consistent across all back-ends
+ * – Decode any existing URI encoding
+ * – Strip directory components
+ * – Replace characters that are not alphanum, dash, dot, or underscore with `_`
+ * – Convert spaces to underscores to avoid unintended encoding by some SDKs
+ *
+ * @param {string} raw The raw filename/path/URL component
+ * @returns {string} A sanitized filename suitable for Azure, GCS, local FS, etc.
+ */
+export function sanitizeFilename(raw = '') {
+    let name = raw;
+    try {
+        name = decodeURIComponent(name);
+    } catch (_) {
+        // Already decoded / not URI encoded – ignore
+    }
+
+    name = path.basename(name);
+    // Replace spaces first so they don't become %20 anywhere
+    name = name.replace(/\s+/g, '_');
+    // Replace any remaining invalid characters
+    name = name.replace(/[^\w\-\.]/g, '_');
+
+    return name;
+} 

package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js

@@ -0,0 +1,144 @@
+import fs from 'fs/promises';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import test from 'ava';
+import axios from 'axios';
+import XLSX from 'xlsx';
+import { FileConversionService } from '../src/services/FileConversionService.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Mock context
+const mockContext = {
+    log: console.log
+};
+
+// Setup: Create test documents
+test.before(async (t) => {
+    const testDir = join(__dirname, 'test-docs');
+    await fs.mkdir(testDir, { recursive: true });
+
+    // Create various test files
+    const textFile = join(testDir, 'test.txt');
+    const largeTextFile = join(testDir, 'large.txt');
+    const unicodeFile = join(testDir, 'unicode.txt');
+    const jsonFile = join(testDir, 'test.json');
+    const emptyFile = join(testDir, 'empty.txt');
+    const excelFile = join(testDir, 'test.xlsx');
+
+    // Regular text content
+    await fs.writeFile(
+        textFile,
+        'This is a test document content.\nIt has multiple lines.\nThird line here.',
+    );
+
+    // Large text content (>100KB)
+    const largeContent = 'Lorem ipsum '.repeat(10000);
+    await fs.writeFile(largeTextFile, largeContent);
+
+    // Unicode content
+    const unicodeContent =
+    '这是中文内容\nこれは日本語です\nЭто русский текст\n🌟 emoji test';
+    await fs.writeFile(unicodeFile, unicodeContent);
+
+    // JSON content
+    await fs.writeFile(jsonFile, JSON.stringify({ test: 'content' }));
+
+    // Empty file
+    await fs.writeFile(emptyFile, '');
+
+    // Create a test Excel file
+    const workbook = XLSX.utils.book_new();
+    const ws1 = XLSX.utils.aoa_to_sheet([
+        ['Header 1', 'Header 2'],
+        ['Data 1', 'Data 2'],
+        ['Data 3', 'Data 4']
+    ]);
+    XLSX.utils.book_append_sheet(workbook, ws1, 'Sheet1');
+    XLSX.writeFile(workbook, excelFile);
+
+    t.context = {
+        testDir,
+        textFile,
+        largeTextFile,
+        unicodeFile,
+        jsonFile,
+        emptyFile,
+        excelFile
+    };
+});
+
+// Cleanup
+test.after.always(async (t) => {
+    await fs.rm(t.context.testDir, { recursive: true, force: true });
+});
+
+// Test Excel to CSV conversion
+test('converts Excel to CSV successfully', async (t) => {
+    const service = new FileConversionService(mockContext);
+    const result = await service.convertFile(t.context.excelFile);
+    
+    t.true(result.converted);
+    t.true(result.convertedPath.endsWith('.csv'));
+    
+    // Read the converted file and verify content
+    const content = await fs.readFile(result.convertedPath, 'utf-8');
+    t.true(content.includes('Header 1,Header 2'));
+    t.true(content.includes('Data 1,Data 2'));
+    t.true(content.includes('Data 3,Data 4'));
+});
+
+// Test document conversion with MarkItDown API
+test('converts document to markdown via MarkItDown API', async (t) => {
+    // Mock axios.get for MarkItDown API
+    const originalAxiosGet = axios.get;
+    axios.get = async (url) => {
+        if (url.includes('test.docx')) {
+            return {
+                data: {
+                    markdown: '# Test Document\n\nThis is a test document converted to markdown.'
+                }
+            };
+        }
+        throw new Error('Invalid URL');
+    };
+
+    const service = new FileConversionService(mockContext);
+    const result = await service.convertFile('test.docx', 'https://example.com/test.docx');
+    
+    t.true(result.converted);
+    t.true(result.convertedPath.endsWith('.md'));
+    
+    // Read the converted file and verify content
+    const content = await fs.readFile(result.convertedPath, 'utf-8');
+    t.true(content.includes('# Test Document'));
+    t.true(content.includes('This is a test document converted to markdown'));
+
+    // Restore original axios.get
+    axios.get = originalAxiosGet;
+});
+
+// Test error handling for missing original URL
+test('handles missing original URL for document conversion', async (t) => {
+    const service = new FileConversionService(mockContext);
+    await t.throwsAsync(
+        async () => service.convertFile('test.docx'),
+        { message: 'Original URL is required for document conversion' }
+    );
+});
+
+// Test error handling for unsupported file types
+test('handles unsupported file types', async (t) => {
+    const service = new FileConversionService(mockContext);
+    const result = await service.convertFile(t.context.jsonFile);
+    t.false(result.converted);
+});
+
+// Test file extension detection
+test('correctly detects file extensions', (t) => {
+    const service = new FileConversionService(mockContext);
+    t.true(service.needsConversion('test.docx'));
+    t.true(service.needsConversion('test.xlsx'));
+    t.false(service.needsConversion('test.txt'));
+    t.false(service.needsConversion('test.json'));
+});