@aj-archipelago/cortex 1.3.62 → 1.3.63

package/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js

@@ -0,0 +1,229 @@
+import test from "ava";
+import { 
+  AZURE_STORAGE_CONTAINER_NAMES, 
+  DEFAULT_AZURE_STORAGE_CONTAINER_NAME,
+  isValidContainerName 
+} from "../src/blobHandler.js";
+
+// Mock environment variables for testing
+const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+
+test.beforeEach(() => {
+  // Reset to original environment
+  process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+});
+
+test.afterEach(() => {
+  // Restore original environment
+  process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+});
+
+test("parseContainerNames should handle single container name", (t) => {
+  // Set environment variable for single container
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "single-container";
+  
+  // We need to reload the module to pick up the new environment variable
+  // Since we can't easily reload ES modules, we'll test the logic directly
+  const parseContainerNames = () => {
+    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
+    return containerStr.split(',').map(name => name.trim());
+  };
+  
+  const result = parseContainerNames();
+  
+  t.is(result.length, 1);
+  t.is(result[0], "single-container");
+});
+
+test("parseContainerNames should handle comma-separated container names", (t) => {
+  // Set environment variable for multiple containers
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "container1,container2,container3";
+  
+  const parseContainerNames = () => {
+    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
+    return containerStr.split(',').map(name => name.trim());
+  };
+  
+  const result = parseContainerNames();
+  
+  t.is(result.length, 3);
+  t.is(result[0], "container1");
+  t.is(result[1], "container2");
+  t.is(result[2], "container3");
+});
+
+test("parseContainerNames should handle comma-separated names with whitespace", (t) => {
+  // Set environment variable with spaces around commas
+  process.env.AZURE_STORAGE_CONTAINER_NAME = " container1 , container2 , container3 ";
+  
+  const parseContainerNames = () => {
+    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
+    return containerStr.split(',').map(name => name.trim());
+  };
+  
+  const result = parseContainerNames();
+  
+  t.is(result.length, 3);
+  t.is(result[0], "container1");
+  t.is(result[1], "container2");
+  t.is(result[2], "container3");
+});
+
+test("parseContainerNames should default to whispertempfiles when env var is not set", (t) => {
+  // Unset environment variable
+  delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+  
+  const parseContainerNames = () => {
+    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
+    return containerStr.split(',').map(name => name.trim());
+  };
+  
+  const result = parseContainerNames();
+  
+  t.is(result.length, 1);
+  t.is(result[0], "whispertempfiles");
+});
+
+test("parseContainerNames should handle empty string", (t) => {
+  // Set environment variable to empty string
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "";
+  
+  const parseContainerNames = () => {
+    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
+    return containerStr.split(',').map(name => name.trim());
+  };
+  
+  const result = parseContainerNames();
+  
+  t.is(result.length, 1);
+  t.is(result[0], "whispertempfiles");
+});
+
+test("parseContainerNames should handle only commas", (t) => {
+  // Set environment variable to only commas
+  process.env.AZURE_STORAGE_CONTAINER_NAME = ",,,";
+  
+  const parseContainerNames = () => {
+    const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
+    return containerStr.split(',').map(name => name.trim());
+  };
+  
+  const result = parseContainerNames();
+  
+  // Should result in 4 empty strings after trimming
+  t.is(result.length, 4);
+  t.is(result[0], "");
+  t.is(result[1], "");
+  t.is(result[2], "");
+  t.is(result[3], "");
+});
+
+test("DEFAULT_AZURE_STORAGE_CONTAINER_NAME should be the first container in the list", (t) => {
+  // Test with current module exports (these are loaded at import time)
+  // The default should be the first item in the array
+  t.is(DEFAULT_AZURE_STORAGE_CONTAINER_NAME, AZURE_STORAGE_CONTAINER_NAMES[0]);
+  
+  // Additional validation that it's a non-empty string
+  t.truthy(DEFAULT_AZURE_STORAGE_CONTAINER_NAME);
+  t.is(typeof DEFAULT_AZURE_STORAGE_CONTAINER_NAME, 'string');
+  t.true(DEFAULT_AZURE_STORAGE_CONTAINER_NAME.length > 0);
+});
+
+test("isValidContainerName should return true for valid container names", (t) => {
+  // Test with each container name from the current configuration
+  AZURE_STORAGE_CONTAINER_NAMES.forEach(containerName => {
+    t.true(isValidContainerName(containerName), `Container name '${containerName}' should be valid`);
+  });
+});
+
+test("isValidContainerName should return false for invalid container names", (t) => {
+  const invalidNames = [
+    "invalid-container",
+    "not-in-list",
+    "",
+    null,
+    undefined,
+    "container-that-does-not-exist"
+  ];
+  
+  invalidNames.forEach(containerName => {
+    t.false(isValidContainerName(containerName), `Container name '${containerName}' should be invalid`);
+  });
+});
+
+test("isValidContainerName should handle edge cases", (t) => {
+  // Test with various edge cases
+  t.false(isValidContainerName(null));
+  t.false(isValidContainerName(undefined));
+  t.false(isValidContainerName(""));
+  t.false(isValidContainerName("   ")); // whitespace only
+  t.false(isValidContainerName(123)); // number
+  t.false(isValidContainerName({})); // object
+  t.false(isValidContainerName([])); // array
+});
+
+test("container configuration should have at least one container", (t) => {
+  t.true(AZURE_STORAGE_CONTAINER_NAMES.length > 0, "Should have at least one container configured");
+  t.truthy(AZURE_STORAGE_CONTAINER_NAMES[0], "First container should not be empty");
+});
+
+test("all configured container names should be non-empty strings", (t) => {
+  AZURE_STORAGE_CONTAINER_NAMES.forEach((containerName, index) => {
+    t.is(typeof containerName, 'string', `Container at index ${index} should be a string`);
+    t.true(containerName.length > 0, `Container at index ${index} should not be empty`);
+    t.true(containerName.trim().length > 0, `Container at index ${index} should not be only whitespace`);
+  });
+});
+
+test("container names should not contain duplicates", (t) => {
+  const uniqueNames = new Set(AZURE_STORAGE_CONTAINER_NAMES);
+  t.is(uniqueNames.size, AZURE_STORAGE_CONTAINER_NAMES.length, "Container names should be unique");
+});
+
+// Integration test with actual environment simulation
+test("complete container parsing workflow", (t) => {
+  const testCases = [
+    {
+      env: "single",
+      expected: ["single"],
+      description: "single container name"
+    },
+    {
+      env: "first,second",
+      expected: ["first", "second"],
+      description: "two container names"
+    },
+    {
+      env: "one, two, three",
+      expected: ["one", "two", "three"],
+      description: "three container names with spaces"
+    },
+    {
+      env: "  leading  ,  trailing  ",
+      expected: ["leading", "trailing"],
+      description: "names with leading/trailing spaces"
+    }
+  ];
+  
+  testCases.forEach(({ env, expected, description }) => {
+    const parseContainerNames = () => {
+      return env.split(',').map(name => name.trim());
+    };
+    
+    const containerNames = parseContainerNames();
+    const defaultContainer = containerNames[0];
+    
+    // Test parsing
+    t.deepEqual(containerNames, expected, `Parsing should work for ${description}`);
+    
+    // Test default is first
+    t.is(defaultContainer, expected[0], `Default should be first container for ${description}`);
+    
+    // Test validation
+    expected.forEach(name => {
+      t.true(containerNames.includes(name), `${name} should be valid for ${description}`);
+    });
+    
+    t.false(containerNames.includes("invalid"), `Invalid name should not be valid for ${description}`);
+  });
+});

package/helper-apps/cortex-file-handler/tests/containerParameterFlow.test.js

@@ -0,0 +1,392 @@
+import test from "ava";
+import fs from "fs";
+import path from "path";
+import os from "os";
+import { fileURLToPath } from "url";
+import { v4 as uuidv4 } from "uuid";
+import axios from "axios";
+import FormData from "form-data";
+import { port } from "../src/start.js";
+import {
+  uploadBlob,
+  isValidContainerName,
+  AZURE_STORAGE_CONTAINER_NAMES,
+  DEFAULT_AZURE_STORAGE_CONTAINER_NAME,
+} from "../src/blobHandler.js";
+import CortexFileHandler from "../src/index.js";
+import {
+  startTestServer,
+  stopTestServer,
+  setupTestDirectory,
+  cleanupHashAndFile,
+  getFolderNameFromUrl,
+} from "./testUtils.helper.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
+
+// Helper function to create test files
+async function createTestFile(content, extension) {
+  const testDir = path.join(__dirname, "test-files");
+  if (!fs.existsSync(testDir)) {
+    fs.mkdirSync(testDir, { recursive: true });
+  }
+  const filename = path.join(testDir, `${uuidv4()}.${extension}`);
+  fs.writeFileSync(filename, content);
+  return filename;
+}
+
+// Mock context for testing
+const mockContext = {
+  log: (message) => console.log(`[TEST] ${message}`),
+  res: null,
+};
+
+// Setup: Create test directory and start server
+test.before(async (t) => {
+  await startTestServer();
+  await setupTestDirectory(t);
+});
+
+// Cleanup
+test.after.always(async (t) => {
+  await stopTestServer();
+
+  // Clean up test directory
+  if (t.context?.testDir) {
+    await fs.promises.rm(t.context.testDir, { recursive: true, force: true });
+  }
+
+  // Clean up any remaining files in the test-files directory
+  const testFilesDir = path.join(__dirname, "test-files");
+  if (fs.existsSync(testFilesDir)) {
+    try {
+      await fs.promises.rm(testFilesDir, { recursive: true, force: true });
+    } catch (error) {
+      console.log("Error cleaning test files:", error);
+    }
+  }
+});
+
+// Test container parameter validation
+test("should validate container names correctly", (t) => {
+  // Test with valid container names from configuration
+  AZURE_STORAGE_CONTAINER_NAMES.forEach(containerName => {
+    t.true(isValidContainerName(containerName), `${containerName} should be valid`);
+  });
+
+  // Test with invalid container names
+  const invalidNames = ["invalid-container", "", null, undefined, "nonexistent"];
+  invalidNames.forEach(name => {
+    t.false(isValidContainerName(name), `${name} should be invalid`);
+  });
+});
+
+// Test uploadBlob function with container parameter
+test("uploadBlob should accept and use container parameter from function parameter", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  // Create a test file
+  const testFile = await createTestFile("test content", "txt");
+  const testStream = fs.createReadStream(testFile);
+  
+  // Mock request with container parameter in function call
+  const mockReq = {
+    headers: { "content-type": "application/octet-stream" },
+  };
+
+  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+
+  try {
+    // Call uploadBlob with container parameter
+    const result = await uploadBlob(
+      mockContext,
+      mockReq,
+      false, // saveToLocal
+      testFile, // filePath
+      null, // hash
+      "test2" // container parameter
+    );
+
+    t.truthy(result);
+    t.truthy(result.url || mockContext.res?.body?.url);
+
+    // Cleanup
+    const uploadedUrl = result.url || mockContext.res?.body?.url;
+    if (uploadedUrl) {
+      const folderName = getFolderNameFromUrl(uploadedUrl);
+      await cleanupHashAndFile(null, uploadedUrl, baseUrl);
+    }
+  } finally {
+    // Restore environment
+    if (originalEnv) {
+      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+    } else {
+      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+    }
+    
+    // Cleanup test file
+    if (fs.existsSync(testFile)) {
+      fs.unlinkSync(testFile);
+    }
+  }
+});
+
+// Test uploadBlob function with container parameter in form data
+test("uploadBlob should accept container parameter from form data", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+
+  try {
+    // Create a test file
+    const testContent = "test content for form data";
+    const testFile = await createTestFile(testContent, "txt");
+
+    // Create form data with container parameter
+    const form = new FormData();
+    form.append("file", fs.createReadStream(testFile), "test.txt");
+    form.append("container", "test3");
+
+    const response = await axios.post(baseUrl, form, {
+      headers: {
+        ...form.getHeaders(),
+        "Content-Type": "multipart/form-data",
+      },
+      validateStatus: (status) => true,
+      timeout: 30000,
+    });
+
+    t.is(response.status, 200);
+    t.truthy(response.data.url);
+
+    // Cleanup
+    await cleanupHashAndFile(null, response.data.url, baseUrl);
+  } finally {
+    // Restore environment
+    if (originalEnv) {
+      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+    } else {
+      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+    }
+  }
+});
+
+// Test invalid container name in form data
+test("uploadBlob should reject invalid container names in form data", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  // Create a test file
+  const testContent = "test content";
+  const testFile = await createTestFile(testContent, "txt");
+
+  try {
+    // Create form data with invalid container parameter
+    const form = new FormData();
+    form.append("file", fs.createReadStream(testFile), "test.txt");
+    form.append("container", "invalid-container-name");
+
+    const response = await axios.post(baseUrl, form, {
+      headers: {
+        ...form.getHeaders(),
+        "Content-Type": "multipart/form-data",
+      },
+      validateStatus: (status) => true,
+      timeout: 30000,
+    });
+
+    t.is(response.status, 400);
+    t.truthy(response.data);
+    t.true(
+      response.data.includes("Invalid container name")
+    );
+  } finally {
+    // Cleanup test file
+    if (fs.existsSync(testFile)) {
+      fs.unlinkSync(testFile);
+    }
+  }
+});
+
+// Test container parameter flow through index.js
+test("CortexFileHandler should pass container parameter for remote file downloads", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+
+  try {
+    // Create a test file URL (this would typically be a remote URL)
+    const testUrl = "https://example.com/test.txt";
+    
+    // Mock request for remote file with container parameter
+    const mockReq = {
+      query: {
+        fetch: testUrl,
+        container: "test2"
+      },
+      method: "GET"
+    };
+
+    // Since we can't easily test real remote downloads in unit tests,
+    // we'll test the parameter extraction and validation
+    const { container } = mockReq.query;
+    
+    t.is(container, "test2");
+    t.true(isValidContainerName(container));
+  } finally {
+    // Restore environment
+    if (originalEnv) {
+      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+    } else {
+      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+    }
+  }
+});
+
+// Test container parameter flow for document processing
+test("CortexFileHandler should pass container parameter for document processing", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+
+  try {
+    // Mock request for document processing with container parameter
+    const mockReq = {
+      body: {
+        params: {
+          uri: "https://example.com/test.pdf",
+          requestId: "test-request",
+          container: "test3"
+        }
+      },
+      query: {},
+      method: "GET"
+    };
+
+    // Extract parameters like index.js does
+    const {
+      uri,
+      requestId,
+      container,
+    } = mockReq.body?.params || mockReq.query;
+
+    t.is(uri, "https://example.com/test.pdf");
+    t.is(requestId, "test-request");
+    t.is(container, "test3");
+    t.true(isValidContainerName(container));
+  } finally {
+    // Restore environment
+    if (originalEnv) {
+      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+    } else {
+      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+    }
+  }
+});
+
+// Test default container behavior
+test("should use default container when no container specified", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  // Test that default container is first in the list
+  t.is(DEFAULT_AZURE_STORAGE_CONTAINER_NAME, AZURE_STORAGE_CONTAINER_NAMES[0]);
+  t.true(isValidContainerName(DEFAULT_AZURE_STORAGE_CONTAINER_NAME));
+
+  // Create a test file
+  const testContent = "test content for default container";
+  const testFile = await createTestFile(testContent, "txt");
+
+  try {
+    // Create form data without container parameter (should use default)
+    const form = new FormData();
+    form.append("file", fs.createReadStream(testFile), "test.txt");
+
+    const response = await axios.post(baseUrl, form, {
+      headers: {
+        ...form.getHeaders(),
+        "Content-Type": "multipart/form-data",
+      },
+      validateStatus: (status) => true,
+      timeout: 30000,
+    });
+
+    t.is(response.status, 200);
+    t.truthy(response.data.url);
+
+    // Cleanup
+    await cleanupHashAndFile(null, response.data.url, baseUrl);
+  } finally {
+    // Cleanup test file
+    if (fs.existsSync(testFile)) {
+      fs.unlinkSync(testFile);
+    }
+  }
+});
+
+// Test container parameter with media chunking
+test("should pass container parameter for media file chunking", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+
+  try {
+    // Mock request for media file processing with container parameter
+    const mockReq = {
+      body: {
+        params: {
+          uri: "https://example.com/test.mp3",
+          requestId: "test-media-request",
+          container: "test1"
+        }
+      },
+      query: {},
+      method: "GET"
+    };
+
+    // Extract parameters like index.js does
+    const {
+      uri,
+      requestId,
+      container,
+    } = mockReq.body?.params || mockReq.query;
+
+    t.is(uri, "https://example.com/test.mp3");
+    t.is(requestId, "test-media-request");
+    t.is(container, "test1");
+    t.true(isValidContainerName(container));
+  } finally {
+    // Restore environment
+    if (originalEnv) {
+      process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+    } else {
+      delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+    }
+  }
+});

package/helper-apps/cortex-file-handler/tests/conversionResilience.test.js

@@ -8,7 +8,7 @@ import XLSX from "xlsx";
 import { port } from "../src/start.js";
 import { gcs, GCS_BUCKETNAME } from "../src/blobHandler.js";
 import { getFileStoreMap, setFileStoreMap } from "../src/redis.js";
-import { cleanupHashAndFile } from "./testUtils.helper.js";
+import { cleanupHashAndFile, startTestServer, stopTestServer } from "./testUtils.helper.js";
 import { gcsUrlExists } from "../src/blobHandler.js";
 
 const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
@@ -47,13 +47,18 @@ function isGcsConfigured() {
   return !!gcs;
 }
 
+// Ensure server is ready before tests
 test.before(async (t) => {
+  await startTestServer();
   const dir = path.join(fs.mkdtempSync(path.join(process.cwd(), "conv-test-")));
   t.context.tmpDir = dir;
 });
 
 test.after.always(async (t) => {
-  fs.rmSync(t.context.tmpDir, { recursive: true, force: true });
+  await stopTestServer();
+  if (t.context?.tmpDir) {
+    fs.rmSync(t.context.tmpDir, { recursive: true, force: true });
+  }
 });
 
 // 1. Remote-URL upload path should still return converted info