@aj-archipelago/cortex 1.3.62 → 1.3.63

package/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js

@@ -1,6 +1,7 @@
 import test from "ava";
 import { StorageService } from "../../src/services/storage/StorageService.js";
 import { StorageFactory } from "../../src/services/storage/StorageFactory.js";
+import { getFileStoreMap, setFileStoreMap, removeFromFileStoreMap } from "../../src/redis.js";
 import path from "path";
 import os from "os";
 import fs from "fs";
@@ -46,20 +47,30 @@ test("should upload file to primary storage", async (t) => {
 test("should upload file to backup storage", async (t) => {
   const factory = new StorageFactory();
   const service = new StorageService(factory);
-  const provider = service.getBackupProvider();
+  const provider = await service.getBackupProvider();
   if (!provider) {
-    t.log("GCS not configured, skipping test");
+    t.log("Backup provider not configured, skipping test");
     t.pass();
     return;
   }
-  const testContent = "test content";
-  const buffer = Buffer.from(testContent);
+  
+  try {
+    const testContent = "test content";
+    const buffer = Buffer.from(testContent);
 
-  const result = await service.uploadFileToBackup(buffer, "test.txt");
-  t.truthy(result.url);
+    const result = await service.uploadFileToBackup(buffer, "test.txt");
+    t.truthy(result.url);
 
-  // Cleanup
-  await service.deleteFileFromBackup(result.url);
+    // Cleanup
+    await service.deleteFileFromBackup(result.url);
+  } catch (error) {
+    if (error.message === "Backup provider not configured") {
+      t.log("Backup provider not configured, skipping test");
+      t.pass();
+    } else {
+      throw error;
+    }
+  }
 });
 
 test("should download file from primary storage", async (t) => {
@@ -82,32 +93,387 @@ test("should download file from primary storage", async (t) => {
 test("should download file from backup storage", async (t) => {
   const factory = new StorageFactory();
   const service = new StorageService(factory);
-  const provider = service.getBackupProvider();
+  const provider = await service.getBackupProvider();
   if (!provider) {
-    t.log("GCS not configured, skipping test");
+    t.log("Backup provider not configured, skipping test");
     t.pass();
     return;
   }
-  const testContent = "test content";
+  
+  try {
+    const testContent = "test content";
+    const buffer = Buffer.from(testContent);
+
+    // Upload first
+    const uploadResult = await service.uploadFileToBackup(buffer, "test.txt");
+
+    // Create temp file for download
+    const tempFile = path.join(os.tmpdir(), "test-download.txt");
+    try {
+      // Download
+      await service.downloadFileFromBackup(uploadResult.url, tempFile);
+      const downloadedContent = await fs.promises.readFile(tempFile);
+      t.deepEqual(downloadedContent, buffer);
+
+      // Cleanup
+      await service.deleteFileFromBackup(uploadResult.url);
+    } finally {
+      // Cleanup temp file
+      if (fs.existsSync(tempFile)) {
+        fs.unlinkSync(tempFile);
+      }
+    }
+  } catch (error) {
+    if (error.message === "Backup provider not configured") {
+      t.log("Backup provider not configured, skipping test");
+      t.pass();
+    } else {
+      throw error;
+    }
+  }
+});
+
+test("should delete file by hash", async (t) => {
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  const testContent = "test content for hash deletion";
   const buffer = Buffer.from(testContent);
+  const testHash = "test-hash-123";
 
-  // Upload first
-  const uploadResult = await service.uploadFileToBackup(buffer, "test.txt");
+  try {
+    // Upload file first
+    const uploadResult = await service.uploadFile(buffer, "test-hash-delete.txt");
+    t.truthy(uploadResult.url);
+
+    // Store file info in Redis map
+    const fileInfo = {
+      url: uploadResult.url,
+      filename: "test-hash-delete.txt",
+      hash: testHash,
+      timestamp: new Date().toISOString()
+    };
+    await setFileStoreMap(testHash, fileInfo);
+
+    // Verify file exists in map
+    const storedInfo = await getFileStoreMap(testHash);
+    t.truthy(storedInfo);
+    t.is(storedInfo.url, uploadResult.url);
+
+    // Delete file by hash
+    const deleteResult = await service.deleteFileByHash(testHash);
+    t.truthy(deleteResult);
+    t.is(deleteResult.hash, testHash);
+    t.is(deleteResult.filename, "test-hash-delete.txt");
+    t.truthy(deleteResult.deleted);
+    t.true(Array.isArray(deleteResult.deleted));
+
+    // Verify file is removed from Redis map
+    const removedInfo = await getFileStoreMap(testHash);
+    t.falsy(removedInfo);
+
+  } catch (error) {
+    // Cleanup in case of error
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (cleanupError) {
+      // Ignore cleanup errors
+    }
+    throw error;
+  }
+});
+
+test("should handle delete file by hash when file not found", async (t) => {
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  const nonExistentHash = "non-existent-hash-456";
+
+  try {
+    await service.deleteFileByHash(nonExistentHash);
+    t.fail("Should have thrown an error for non-existent hash");
+  } catch (error) {
+    t.true(error.message.includes("not found"));
+  }
+});
+
+test("should handle delete file by hash with missing hash parameter", async (t) => {
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+
+  try {
+    await service.deleteFileByHash("");
+    t.fail("Should have thrown an error for empty hash");
+  } catch (error) {
+    t.true(error.message.includes("Missing hash parameter"));
+  }
+
+  try {
+    await service.deleteFileByHash(null);
+    t.fail("Should have thrown an error for null hash");
+  } catch (error) {
+    t.true(error.message.includes("Missing hash parameter"));
+  }
+
+  try {
+    await service.deleteFileByHash(undefined);
+    t.fail("Should have thrown an error for undefined hash");
+  } catch (error) {
+    t.true(error.message.includes("Missing hash parameter"));
+  }
+});
+
+test("should delete file by hash with backup storage", async (t) => {
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  const testContent = "test content for backup deletion";
+  const buffer = Buffer.from(testContent);
+  const testHash = "test-hash-backup-456";
+
+  try {
+    // Upload file first
+    const uploadResult = await service.uploadFile(buffer, "test-backup-delete.txt");
+    t.truthy(uploadResult.url);
+
+    // Store file info in Redis map with backup URL
+    const fileInfo = {
+      url: uploadResult.url,
+      gcs: "gs://test-bucket/test-backup-file.txt", // Mock backup URL
+      filename: "test-backup-delete.txt",
+      hash: testHash,
+      timestamp: new Date().toISOString()
+    };
+    await setFileStoreMap(testHash, fileInfo);
+
+    // Delete file by hash
+    const deleteResult = await service.deleteFileByHash(testHash);
+    t.truthy(deleteResult);
+    t.is(deleteResult.hash, testHash);
+    t.is(deleteResult.filename, "test-backup-delete.txt");
+    t.truthy(deleteResult.deleted);
+    t.true(Array.isArray(deleteResult.deleted));
+
+    // Should have attempted both primary and backup deletion
+    const deletionResults = deleteResult.deleted;
+    t.true(deletionResults.length >= 1, "Should have at least primary deletion result");
+
+    // Verify file is removed from Redis map
+    const removedInfo = await getFileStoreMap(testHash);
+    t.falsy(removedInfo);
+
+  } catch (error) {
+    // Cleanup in case of error
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (cleanupError) {
+      // Ignore cleanup errors
+    }
+    throw error;
+  }
+});
+
+test("should handle delete file by hash when Redis map is corrupted", async (t) => {
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  const testHash = "test-hash-corrupted-789";
+
+  try {
+    // Store corrupted data in Redis map
+    const corruptedInfo = {
+      // Missing required fields like url
+      corrupted: true,
+      timestamp: new Date().toISOString()
+    };
+    await setFileStoreMap(testHash, corruptedInfo);
+
+    // Delete file by hash should handle corrupted data gracefully
+    const deleteResult = await service.deleteFileByHash(testHash);
+    t.truthy(deleteResult);
+    t.is(deleteResult.hash, testHash);
+    t.truthy(deleteResult.deleted);
+    t.true(Array.isArray(deleteResult.deleted));
+
+    // Should have removed the corrupted entry from Redis
+    const removedInfo = await getFileStoreMap(testHash);
+    t.falsy(removedInfo);
+
+  } catch (error) {
+    // Cleanup in case of error
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (cleanupError) {
+      // Ignore cleanup errors
+    }
+    throw error;
+  }
+});
+
+test("should handle delete file by hash with empty URL in Redis", async (t) => {
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  const testHash = "test-hash-empty-url-654";
 
-  // Create temp file for download
-  const tempFile = path.join(os.tmpdir(), "test-download.txt");
   try {
-    // Download
-    await service.downloadFileFromBackup(uploadResult.url, tempFile);
-    const downloadedContent = await fs.promises.readFile(tempFile);
-    t.deepEqual(downloadedContent, buffer);
+    // Store file info in Redis map with empty/null URL
+    const fileInfo = {
+      url: null,
+      filename: "test-empty-url-delete.txt",
+      hash: testHash,
+      timestamp: new Date().toISOString()
+    };
+    await setFileStoreMap(testHash, fileInfo);
+
+    // Verify the hash exists in Redis before deletion (skip lazy cleanup)
+    const storedInfo = await getFileStoreMap(testHash, true);
+    t.truthy(storedInfo, "Hash should exist in Redis before deletion");
+
+    // Delete file by hash - should handle missing URL gracefully
+    const deleteResult = await service.deleteFileByHash(testHash);
+    t.truthy(deleteResult);
+    t.is(deleteResult.hash, testHash);
+    t.is(deleteResult.filename, "test-empty-url-delete.txt");
+    t.truthy(deleteResult.deleted);
+    t.true(Array.isArray(deleteResult.deleted));
+
+    // Should still remove from Redis map even if no actual file to delete
+    const removedInfo = await getFileStoreMap(testHash);
+    t.falsy(removedInfo);
+
+  } catch (error) {
+    // Cleanup in case of error
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (cleanupError) {
+      // Ignore cleanup errors
+    }
+    throw error;
+  }
+});
 
+// Container-specific tests
+test("should upload file with specific container name", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  
+  // Create a temporary file
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-"));
+  const testFile = path.join(tempDir, "test.txt");
+  fs.writeFileSync(testFile, "test content");
+  
+  try {
+    // Mock environment to have multiple containers
+    const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+    process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+    
+    try {
+      // Test upload with specific container
+      const result = await service.uploadFileWithProviders(
+        { log: () => {} }, // mock context
+        testFile,
+        "test-request",
+        null,
+        "test2"
+      );
+      
+      t.truthy(result.url);
+      t.truthy(result.url.includes("test2") || result.url.includes("/test2/"));
+      
+      // Cleanup
+      await service.deleteFiles("test-request");
+    } finally {
+      // Restore original env
+      if (originalEnv) {
+        process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+      } else {
+        delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+      }
+    }
+  } finally {
+    // Cleanup temp file
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+test("should use default container when no container specified", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  
+  // Create a temporary file
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-"));
+  const testFile = path.join(tempDir, "test.txt");
+  fs.writeFileSync(testFile, "test content");
+  
+  try {
+    // Test upload without container (should use default)
+    const result = await service.uploadFileWithProviders(
+      { log: () => {} }, // mock context
+      testFile,
+      "test-request",
+      null,
+      null // no container specified
+    );
+    
+    t.truthy(result.url);
+    
     // Cleanup
-    await service.deleteFileFromBackup(uploadResult.url);
+    await service.deleteFiles("test-request");
   } finally {
     // Cleanup temp file
-    if (fs.existsSync(tempFile)) {
-      fs.unlinkSync(tempFile);
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+test("should pass container parameter through uploadFile method", async (t) => {
+  if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
+    t.pass("Skipping test - Azure not configured");
+    return;
+  }
+
+  const factory = new StorageFactory();
+  const service = new StorageService(factory);
+  
+  // Create a temporary file
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-"));
+  const testFile = path.join(tempDir, "test.txt");
+  fs.writeFileSync(testFile, "test content");
+  
+  try {
+    // Mock environment to have multiple containers
+    const originalEnv = process.env.AZURE_STORAGE_CONTAINER_NAME;
+    process.env.AZURE_STORAGE_CONTAINER_NAME = "test1,test2,test3";
+    
+    try {
+      // Test upload using the uploadFile method with container parameter
+      const result = await service.uploadFile(
+        { log: () => {} }, // context
+        testFile,         // filePath
+        "test-request",   // requestId
+        null,             // hash
+        "test3"           // containerName
+      );
+      
+      t.truthy(result.url);
+      
+      // Cleanup
+      await service.deleteFiles("test-request");
+    } finally {
+      // Restore original env
+      if (originalEnv) {
+        process.env.AZURE_STORAGE_CONTAINER_NAME = originalEnv;
+      } else {
+        delete process.env.AZURE_STORAGE_CONTAINER_NAME;
+      }
     }
+  } finally {
+    // Cleanup temp file
+    fs.rmSync(tempDir, { recursive: true, force: true });
   }
 });

package/helper-apps/cortex-file-handler/tests/testUtils.helper.js

@@ -1,6 +1,9 @@
 import axios from "axios";
 import { execSync } from "child_process";
 import fs from "fs/promises";
+import path from "path";
+
+import { app, port } from "../src/start.js";
 
 export async function cleanupHashAndFile(hash, uploadedUrl, baseUrl) {
   // Only perform hash operations if hash is provided
@@ -87,3 +90,74 @@ export async function createTestMediaFile(filepath, durationSeconds = 10) {
     throw error;
   }
 }
+
+// Test server management helpers
+let server;
+
+/**
+ * Starts the test server and waits for it to be ready
+ * @param {Object} options - Optional configuration
+ * @param {Function} options.beforeReady - Optional callback to run after server starts but before ready check
+ * @returns {Promise<Object>} - Server instance
+ */
+export async function startTestServer(options = {}) {
+  if (server) {
+    throw new Error("Test server is already running");
+  }
+
+  // Start the server for tests
+  server = app.listen(port, () => {
+    console.log(`Test server started on port ${port}`);
+  });
+  
+  // Wait for server to be ready
+  await new Promise((resolve) => setTimeout(resolve, 1000));
+
+  // Run any pre-ready setup if provided
+  if (options.beforeReady) {
+    await options.beforeReady();
+  }
+
+  // Verify server is responding
+  try {
+    await axios.get(`http://localhost:${port}/files`);
+  } catch (error) {
+    // 404 is fine, it means server is running but directory is empty
+    if (error.response?.status !== 404) {
+      throw new Error("Server not ready");
+    }
+  }
+
+  return server;
+}
+
+/**
+ * Stops the test server
+ * @param {Function} beforeClose - Optional callback to run before closing server
+ * @returns {Promise<void>}
+ */
+export async function stopTestServer(beforeClose) {
+  if (beforeClose) {
+    await beforeClose();
+  }
+  
+  if (server) {
+    server.close();
+    server = null;
+  }
+}
+
+/**
+ * Creates a test directory and sets up context for AVA tests
+ * @param {Object} t - AVA test context
+ * @param {string} dirName - Optional directory name (defaults to "test-files")
+ * @returns {Promise<string>} - Path to created test directory
+ */
+export async function setupTestDirectory(t, dirName = "test-files") {
+  const testDir = path.join(process.cwd(), "tests", dirName);
+  await fs.mkdir(testDir, { recursive: true });
+  if (t && t.context) {
+    t.context.testDir = testDir;
+  }
+  return testDir;
+}

package/lib/cortexResponse.js

@@ -0,0 +1,153 @@
+class CortexResponse {
+    constructor({
+        output_text = "",
+        output = null,
+        finishReason = 'stop',
+        toolCalls = null,
+        functionCall = null,
+        citations = null,
+        searchQueries = null,
+        searchResults = null,
+        realTimeData = null,
+        artifacts = null,
+        usage = null,
+        metadata = {},
+        error = null
+    } = {}) {
+        this._output_text = output_text;
+        this._output = output;
+        this._finishReason = finishReason;
+        this._toolCalls = toolCalls;
+        this._functionCall = functionCall;
+        this._citations = citations;
+        this._searchQueries = searchQueries;
+        this._searchResults = searchResults;
+        this._realTimeData = realTimeData;
+        this._artifacts = artifacts;
+        this._usage = usage;
+        this._metadata = {
+            timestamp: new Date().toISOString(),
+            ...metadata
+        };
+        this._error = error;
+    }
+
+    // Getters
+    get output_text() { return this._output_text; }
+    get output() { return this._output; }
+    get finishReason() { return this._finishReason; }
+    get toolCalls() { return this._toolCalls; }
+    get tool_calls() { return this._toolCalls; } // For legacy compatibility
+    get functionCall() { return this._functionCall; }
+    get citations() { return this._citations; }
+    get searchQueries() { return this._searchQueries; }
+    get searchResults() { return this._searchResults; }
+    get realTimeData() { return this._realTimeData; }
+    get artifacts() { return this._artifacts; }
+    get usage() { return this._usage; }
+    get metadata() { return this._metadata; }
+    get error() { return this._error; }
+
+    // Setters
+    set output_text(value) { this._output_text = value; }
+    set output(value) { this._output = value; }
+    set finishReason(value) { this._finishReason = value; }
+    set toolCalls(value) { 
+        this._toolCalls = value;
+        if (value && value.length > 0) {
+            this._finishReason = 'tool_calls';
+        }
+    }
+    set functionCall(value) { 
+        this._functionCall = value;
+        if (value) {
+            this._finishReason = 'function_call';
+        }
+    }
+    set citations(value) { this._citations = value; }
+    set searchQueries(value) { this._searchQueries = value; }
+    set searchResults(value) { this._searchResults = value; }
+    set realTimeData(value) { this._realTimeData = value; }
+    set artifacts(value) { this._artifacts = value; }
+    set usage(value) { this._usage = value; }
+    set metadata(value) { this._metadata = { ...this._metadata, ...value }; }
+    set error(value) { this._error = value; }
+
+    // Utility methods
+    hasToolCalls() {
+        return this._toolCalls && this._toolCalls.length > 0;
+    }
+
+    hasCitations() {
+        return this._citations && this._citations.length > 0;
+    }
+
+    hasSearchResults() {
+        return this._searchResults && this._searchResults.length > 0;
+    }
+
+    hasArtifacts() {
+        return this._artifacts && this._artifacts.length > 0;
+    }
+
+    isError() {
+        return this._error !== null;
+    }
+
+    hasOutput() {
+        return this._output && this._output.length > 0;
+    }
+
+    hasOutputText() {
+        return this._output_text && this._output_text.length > 0;
+    }
+
+    // Get text content from output array (OpenAI format)
+    getTextFromOutput() {
+        if (!this._output || !Array.isArray(this._output)) {
+            return this._output_text || "";
+        }
+        
+        // Extract text from output items that have text content
+        const textItems = this._output
+            .filter(item => item && item.text)
+            .map(item => item.text);
+        
+        return textItems.join("");
+    }
+
+
+    // String compatibility methods
+    toString() {
+        return this._output_text;
+    }
+    
+    valueOf() {
+        return this._output_text;
+    }
+    
+    toJSON() {
+        return this._output_text;
+    }
+
+    // Convert to plain object
+    toObject() {
+        return {
+            output_text: this._output_text,
+            output: this._output,
+            finishReason: this._finishReason,
+            toolCalls: this._toolCalls,
+            functionCall: this._functionCall,
+            citations: this._citations,
+            searchQueries: this._searchQueries,
+            searchResults: this._searchResults,
+            realTimeData: this._realTimeData,
+            artifacts: this._artifacts,
+            usage: this._usage,
+            metadata: this._metadata,
+            error: this._error
+        };
+    }
+}
+
+export default CortexResponse;