@aj-archipelago/cortex 1.3.62 → 1.3.63

package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js

@@ -7,25 +7,40 @@ import { generateShortId } from "../../utils/filenameUtils.js";
 export class StorageService {
   constructor(factory) {
     this.factory = factory || new StorageFactory();
-    this.primaryProvider = this.factory.getPrimaryProvider();
-    this.backupProvider = this.factory.getGCSProvider();
+    this.primaryProvider = null;
+    this.backupProvider = null;
+    this._initialized = false;
   }
 
-  getPrimaryProvider() {
+  async _initialize() {
+    if (!this._initialized) {
+      this.primaryProvider = await this.factory.getPrimaryProvider();
+      this.backupProvider = this.factory.getGCSProvider();
+      this._initialized = true;
+    }
+  }
+
+  async getPrimaryProvider() {
+    await this._initialize();
     return this.primaryProvider;
   }
 
-  getBackupProvider() {
+  async getBackupProvider() {
+    await this._initialize();
     return this.backupProvider;
   }
 
+
+
   async uploadFile(...args) {
     /*
             Supported call shapes:
             1) uploadFile(buffer, filename)
-            2) uploadFile(context, filePath, requestId, hash?) – legacy internal use
+            2) uploadFile(context, filePath, requestId, hash?, filename?, containerName?) – legacy internal use
         */
 
+    await this._initialize();
+
     // Shape (buffer, filename)
     if (
       args.length === 2 &&
@@ -50,12 +65,14 @@ export class StorageService {
       }
     }
 
-    // Fallback to legacy (context, filePath, requestId, hash?)
-    const [context, filePath, requestId, hash] = args;
-    return this.uploadFileWithProviders(context, filePath, requestId, hash);
+    // Fallback to legacy (context, filePath, requestId, hash?, filename?, containerName?)
+    const [context, filePath, requestId, hash, filename, containerName] = args;
+    return this.uploadFileWithProviders(context, filePath, requestId, hash, filename, containerName);
   }
 
   async uploadFileToBackup(fileOrBuffer, filename) {
+    await this._initialize();
+    
     if (!this.backupProvider) {
       throw new Error("Backup provider not configured");
     }
@@ -86,6 +103,8 @@ export class StorageService {
   }
 
   async downloadFile(url, destinationPath = null) {
+    await this._initialize();
+    
     const useBackup = url.startsWith("gs://");
 
     if (useBackup && !this.backupProvider) {
@@ -119,6 +138,8 @@ export class StorageService {
   }
 
   async deleteFile(url) {
+    await this._initialize();
+    
     if (typeof this.primaryProvider.deleteFile === "function") {
       return await this.primaryProvider.deleteFile(url);
     }
@@ -127,6 +148,8 @@ export class StorageService {
   }
 
   async deleteFileFromBackup(url) {
+    await this._initialize();
+    
     if (!this.backupProvider) {
       throw new Error("Backup provider not configured");
     }
@@ -137,18 +160,89 @@ export class StorageService {
     return await this.backupProvider.deleteFiles([url]);
   }
 
-  async uploadFileWithProviders(context, filePath, requestId, hash = null) {
-    // Generate filename once to ensure both providers use the same name
-    const fileExtension = path.extname(filePath);
-    const shortId = generateShortId();
-    const filename = `${shortId}${fileExtension}`;
+  /**
+   * Delete a single file by its hash from both primary and backup storage
+   * @param {string} hash - The hash of the file to delete
+   * @returns {Promise<Object>} Object containing deletion results and file info
+   */
+  async deleteFileByHash(hash) {
+    await this._initialize();
+    
+    if (!hash) {
+      throw new Error("Missing hash parameter");
+    }
+
+    const results = [];
+
+    // Get and remove file information from Redis map (non-atomic operations)
+    const { getFileStoreMap, removeFromFileStoreMap } = await import("../../redis.js");
+    const hashResult = await getFileStoreMap(hash);
+    
+    if (hashResult) {
+      await removeFromFileStoreMap(hash);
+    }
+    
+    if (!hashResult) {
+      throw new Error(`File with hash ${hash} not found`);
+    }
+
+    // Delete from primary storage
+    if (hashResult.url) {
+      try {
+        const primaryResult = await this.deleteFile(hashResult.url);
+        if (primaryResult) {
+          results.push({ provider: 'primary', result: primaryResult });
+        }
+      } catch (error) {
+        console.error(`Error deleting file from primary storage:`, error);
+        results.push({ provider: 'primary', error: error.message });
+      }
+    }
+
+    // Delete from backup storage (GCS)
+    if (hashResult.gcs && this.backupProvider) {
+      try {
+        const backupResult = await this.deleteFileFromBackup(hashResult.gcs);
+        if (backupResult) {
+          results.push({ provider: 'backup', result: backupResult });
+        }
+      } catch (error) {
+        console.error(`Error deleting file from backup storage:`, error);
+        results.push({ provider: 'backup', error: error.message });
+      }
+    }
 
-    const primaryResult = await this.primaryProvider.uploadFile(
+    // Note: Hash was already removed from Redis atomically at the beginning
+    // No need to remove again
+
+    return {
+      hash,
+      deleted: results,
+      filename: hashResult.filename
+    };
+  }
+
+  async uploadFileWithProviders(context, filePath, requestId, hash = null, filename = null, containerName = null) {
+    await this._initialize();
+    
+    // Use provided filename or generate one
+    const finalFilename = filename || (() => {
+      const fileExtension = path.extname(filePath);
+      const shortId = generateShortId();
+      return `${shortId}${fileExtension}`;
+    })();
+
+    // Get the appropriate provider for the container
+    const primaryProvider = containerName ? 
+      await this.factory.getAzureProvider(containerName) : 
+      this.primaryProvider;
+
+    const primaryResult = await primaryProvider.uploadFile(
       context,
       filePath,
       requestId,
       hash,
-      filename,
+      finalFilename,
     );
 
     let gcsResult = null;
@@ -158,7 +252,7 @@ export class StorageService {
         filePath,
         requestId,
         hash,
-        filename,
+        finalFilename,
       );
     }
 
@@ -166,6 +260,8 @@ export class StorageService {
   }
 
   async deleteFiles(requestId) {
+    await this._initialize();
+    
     if (!requestId) {
       throw new Error("Missing requestId parameter");
     }
@@ -201,6 +297,8 @@ export class StorageService {
   }
 
   async fileExists(url) {
+    await this._initialize();
+    
     if (!url) {
       return false;
     }
@@ -219,6 +317,8 @@ export class StorageService {
   }
 
   async cleanup(urls) {
+    await this._initialize();
+    
     if (!urls || !urls.length) return;
 
     const results = [];
@@ -251,6 +351,8 @@ export class StorageService {
   }
 
   async ensureGCSUpload(context, existingFile) {
+    await this._initialize();
+    
     if (
       !this.backupProvider ||
       !existingFile.url ||
@@ -293,6 +395,8 @@ export class StorageService {
   }
 
   async downloadFileFromBackup(url, destinationPath = null) {
+    await this._initialize();
+    
     if (!this.backupProvider) {
       throw new Error("Backup provider not configured");
     }

package/helper-apps/cortex-file-handler/src/start.js

@@ -15,17 +15,21 @@ import { publicIpv4 } from "public-ip";
 // a test run.
 
 let ipAddress = "localhost";
-if (process.env.NODE_ENV !== "test") {
-  try {
-    ipAddress = await publicIpv4();
-  } catch (err) {
-    // In rare cases querying the public IP can fail (e.g. no network when
-    // running offline).  Keep the default of "localhost" in that case so we
-    // still generate valid URLs.
-    console.warn(
-      "Unable to determine public IPv4 address – defaulting to 'localhost'.",
-      err,
-    );
+
+// Initialize IP address asynchronously (only for non-test environments)
+async function initializeIpAddress() {
+  if (process.env.NODE_ENV !== "test") {
+    try {
+      ipAddress = await publicIpv4();
+    } catch (err) {
+      // In rare cases querying the public IP can fail (e.g. no network when
+      // running offline).  Keep the default of "localhost" in that case so we
+      // still generate valid URLs.
+      console.warn(
+        "Unable to determine public IPv4 address – defaulting to 'localhost'.",
+        err,
+      );
+    }
   }
 }
 
@@ -82,10 +86,16 @@ app.all("/api/MediaFileChunker", async (req, res) => {
   }
 });
 
-app.listen(port, () => {
-  console.log(
-    `Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`,
-  );
-});
+// Only start the server if this module is being run directly (not imported for tests)
+if (import.meta.url === `file://${process.argv[1]}`) {
+  initializeIpAddress().then(() => {
+    app.listen(port, () => {
+      console.log(
+        `Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`,
+      );
+    });
+  });
+}
+// For tests, we'll keep ipAddress as "localhost" by default - no need to initialize
 
-export { port, publicFolder, ipAddress };
+export { port, publicFolder, ipAddress, app };

package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js

@@ -90,6 +90,10 @@ test("converts Excel to CSV successfully", async (t) => {
 
 // Test document conversion with MarkItDown API
 test("converts document to markdown via MarkItDown API", async (t) => {
+  // Set the environment variable for the test
+  const originalEnv = process.env.MARKITDOWN_CONVERT_URL;
+  process.env.MARKITDOWN_CONVERT_URL = "http://localhost:8080/convert?url=";
+  
   // Mock axios.get for MarkItDown API
   const originalAxiosGet = axios.get;
   axios.get = async (url) => {
@@ -118,8 +122,13 @@ test("converts document to markdown via MarkItDown API", async (t) => {
   t.true(content.includes("# Test Document"));
   t.true(content.includes("This is a test document converted to markdown"));
 
-  // Restore original axios.get
+  // Restore original axios.get and environment variable
   axios.get = originalAxiosGet;
+  if (originalEnv) {
+    process.env.MARKITDOWN_CONVERT_URL = originalEnv;
+  } else {
+    delete process.env.MARKITDOWN_CONVERT_URL;
+  }
 });
 
 // Test error handling for missing original URL
@@ -145,3 +154,45 @@ test("correctly detects file extensions", (t) => {
   t.false(service.needsConversion("test.txt"));
   t.false(service.needsConversion("test.json"));
 });
+
+// Test _saveConvertedFile method signature and container parameter handling
+test("_saveConvertedFile accepts container parameter", async (t) => {
+  const service = new FileConversionService(mockContext, false); // Use local storage for testing
+  
+  // Create a test file
+  const testFile = join(t.context.testDir, "container-param-test.txt");
+  await fs.writeFile(testFile, "Test content for container parameter");
+  
+  // Test that the method accepts all parameters without throwing
+  const result = await service._saveConvertedFile(
+    testFile,
+    "test-request-id",
+    "test-filename.txt",
+    "test-container"
+  );
+  
+  t.truthy(result);
+  t.truthy(result.url);
+  t.true(typeof result.url === 'string');
+});
+
+// Test ensureConvertedVersion method signature with container parameter
+test("ensureConvertedVersion accepts container parameter", async (t) => {
+  const service = new FileConversionService(mockContext, false);
+  
+  // Mock file info object
+  const fileInfo = {
+    url: "http://example.com/test.txt", // Non-convertible file
+    gcs: "gs://bucket/test.txt"
+  };
+  
+  // Test that the method accepts container parameter without throwing
+  const result = await service.ensureConvertedVersion(
+    fileInfo,
+    "test-request-id",
+    "test-container"
+  );
+  
+  t.truthy(result);
+  t.is(result.url, fileInfo.url); // Should return original for non-convertible file
+});

package/helper-apps/cortex-file-handler/tests/blobHandler.test.js

@@ -11,6 +11,9 @@ import {
   gcsUrlExists,
   deleteGCS,
   getBlobClient,
+  AZURE_STORAGE_CONTAINER_NAMES,
+  DEFAULT_AZURE_STORAGE_CONTAINER_NAME,
+  isValidContainerName,
 } from "../src/blobHandler.js";
 import { urlExists } from "../src/helper.js";
 import CortexFileHandler from "../src/index.js";
@@ -314,3 +317,40 @@ test("test hash check returns 404 when both storages are empty", async (t) => {
     }
   }
 });
+
+// Container name parsing and validation tests
+test("AZURE_STORAGE_CONTAINER_NAMES should be an array with at least one container", (t) => {
+  t.true(Array.isArray(AZURE_STORAGE_CONTAINER_NAMES), "Should be an array");
+  t.true(AZURE_STORAGE_CONTAINER_NAMES.length > 0, "Should have at least one container");
+  
+  // All items should be non-empty strings
+  AZURE_STORAGE_CONTAINER_NAMES.forEach((name, index) => {
+    t.is(typeof name, 'string', `Container at index ${index} should be a string`);
+    t.true(name.length > 0, `Container at index ${index} should not be empty`);
+  });
+});
+
+test("DEFAULT_AZURE_STORAGE_CONTAINER_NAME should be the first container", (t) => {
+  t.is(DEFAULT_AZURE_STORAGE_CONTAINER_NAME, AZURE_STORAGE_CONTAINER_NAMES[0]);
+  t.truthy(DEFAULT_AZURE_STORAGE_CONTAINER_NAME);
+  t.is(typeof DEFAULT_AZURE_STORAGE_CONTAINER_NAME, 'string');
+});
+
+test("isValidContainerName should validate container names correctly", (t) => {
+  // All configured containers should be valid
+  AZURE_STORAGE_CONTAINER_NAMES.forEach(containerName => {
+    t.true(isValidContainerName(containerName), `${containerName} should be valid`);
+  });
+  
+  // Invalid containers should return false
+  t.false(isValidContainerName("invalid-container"));
+  t.false(isValidContainerName(""));
+  t.false(isValidContainerName(null));
+  t.false(isValidContainerName(undefined));
+  t.false(isValidContainerName("non-existent"));
+});
+
+test("container names should be unique", (t) => {
+  const uniqueNames = new Set(AZURE_STORAGE_CONTAINER_NAMES);
+  t.is(uniqueNames.size, AZURE_STORAGE_CONTAINER_NAMES.length, "All container names should be unique");
+});