@aj-archipelago/cortex 1.3.17 → 1.3.19

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -494,25 +494,37 @@ async function deleteGCS(blobName) {
   if (!gcs) throw new Error("Google Cloud Storage is not initialized");
 
   try {
+    const bucket = gcs.bucket(GCS_BUCKETNAME);
+    const deletedFiles = [];
+
     if (process.env.STORAGE_EMULATOR_HOST) {
       // For fake GCS server, use HTTP API directly
-      const response = await axios.delete(
-        `http://localhost:4443/storage/v1/b/${GCS_BUCKETNAME}/o/${encodeURIComponent(blobName)}`,
-        { validateStatus: status => status === 200 || status === 404 }
+      const response = await axios.get(
+        `http://localhost:4443/storage/v1/b/${GCS_BUCKETNAME}/o`,
+        { params: { prefix: blobName } }
       );
-      if (response.status === 200) {
-        console.log(`Cleaned GCS file: ${blobName}`);
-        return [blobName];
+      if (response.data.items) {
+        for (const item of response.data.items) {
+          await axios.delete(
+            `http://localhost:4443/storage/v1/b/${GCS_BUCKETNAME}/o/${encodeURIComponent(item.name)}`,
+            { validateStatus: status => status === 200 || status === 404 }
+          );
+          deletedFiles.push(item.name);
+        }
       }
-      return [];
     } else {
       // For real GCS, use the SDK
-      const bucket = gcs.bucket(GCS_BUCKETNAME);
-      const file = bucket.file(blobName);
-      await file.delete();
-      console.log(`Cleaned GCS file: ${blobName}`);
-      return [blobName];
+      const [files] = await bucket.getFiles({ prefix: blobName });
+      for (const file of files) {
+        await file.delete();
+        deletedFiles.push(file.name);
+      }
+    }
+
+    if (deletedFiles.length > 0) {
+      console.log(`Cleaned GCS files: ${deletedFiles.join(', ')}`);
     }
+    return deletedFiles;
   } catch (error) {
     if (error.code !== 404) {
       console.error(`Error in deleteGCS: ${error}`);
@@ -541,4 +553,15 @@ async function ensureGCSUpload(context, existingFile) {
   return existingFile;
 }
 
-export { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs };
+// Helper function to upload a chunk to GCS
+async function uploadChunkToGCS(chunkPath, requestId) {
+    if (!gcs) return null;
+    
+    const gcsFileName = `${requestId}/${path.basename(chunkPath)}`;
+    await gcs.bucket(GCS_BUCKETNAME).upload(chunkPath, {
+        destination: gcsFileName
+    });
+    return `gs://${GCS_BUCKETNAME}/${gcsFileName}`;
+}
+
+export { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, uploadChunkToGCS };

package/helper-apps/cortex-file-handler/index.js

@@ -1,5 +1,5 @@
 import { downloadFile, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, AZURE_STORAGE_CONTAINER_NAME } from './blobHandler.js';
+import { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, AZURE_STORAGE_CONTAINER_NAME, uploadChunkToGCS } from './blobHandler.js';
 import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
 import { ensureEncoded, ensureFileExtension, urlExists } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
@@ -104,9 +104,7 @@ async function CortexFileHandler(context, req) {
         const azureResult = useAzure ? await deleteBlob(deleteRequestId) : await deleteFolder(deleteRequestId);
         const gcsResult = [];
         if (gcs) {
-            for (const blobName of azureResult) {
-                gcsResult.push(...await deleteGCS(blobName));
-            }
+            gcsResult.push(...await deleteGCS(deleteRequestId));
         }
         
         context.res = {
@@ -393,10 +391,21 @@ async function CortexFileHandler(context, req) {
             // sequential processing of chunks
             for (let index = 0; index < chunks.length; index++) {
                 const chunkPath = chunks[index];
-                const blobName = useAzure ? await saveFileToBlob(chunkPath, requestId) : await moveFileToPublicFolder(chunkPath, requestId);
+                let blobName;
+                let gcsUrl;
+                
+                if (useAzure) {
+                    blobName = await saveFileToBlob(chunkPath, requestId);
+                } else {
+                    blobName = await moveFileToPublicFolder(chunkPath, requestId);
+                }
+                
+                // If GCS is configured, save to GCS
+                gcsUrl = await uploadChunkToGCS(chunkPath, requestId);
+                
                 const chunkOffset = chunkOffsets[index];
-                result.push({ uri: blobName, offset: chunkOffset });
-                console.log(`Saved chunk as: ${blobName}`);
+                result.push({ uri: blobName, offset: chunkOffset, gcs: gcsUrl });
+                console.log(`Saved chunk as: ${blobName}${gcsUrl ? ` and ${gcsUrl}` : ''}`);
                 await sendProgress();
             }
 

package/helper-apps/cortex-file-handler/tests/start.test.js

@@ -5,6 +5,10 @@ import axios from 'axios';
 import FormData from 'form-data';
 import { port, publicFolder, ipAddress } from '../start.js';
 import { v4 as uuidv4 } from 'uuid';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+import { execSync } from 'child_process';
 
 // Add these helper functions at the top after imports
 const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
@@ -597,8 +601,7 @@ test.serial('should handle hash reuse with Azure storage', async t => {
     const originalUrl = upload1.data.url;
     
     // Check hash exists and returns the correct URL
-    const hashCheck1 = await axios.get(baseUrl, {
-        params: { hash: testHash, checkHash: true },
+    const hashCheck1 = await axios.get(baseUrl, { hash: testHash, checkHash: true }, {
         validateStatus: status => true
     });
     t.is(hashCheck1.status, 200, 'Hash should exist after first upload');
@@ -634,8 +637,7 @@ test.serial('should handle hash reuse with Azure storage', async t => {
     await cleanupUploadedFile(t, originalUrl);
     
     // Verify hash is now gone
-    const hashCheckAfterDelete = await axios.get(baseUrl, {
-        params: { hash: testHash, checkHash: true },
+    const hashCheckAfterDelete = await axios.get(baseUrl, { hash: testHash, checkHash: true }, {
         validateStatus: status => true
     });
     t.is(hashCheckAfterDelete.status, 404, 'Hash should be gone after file deletion');
@@ -762,6 +764,144 @@ test.serial('should handle GCS URL format and accessibility', async t => {
     await cleanupUploadedFile(t, uploadResponse.data.url);
 });
 
+// Add this helper function after other helper functions
+async function createAndUploadTestFile() {
+    // Create a temporary file path
+    const tempDir = path.join(os.tmpdir(), uuidv4());
+    fs.mkdirSync(tempDir, { recursive: true });
+    const tempFile = path.join(tempDir, 'test.mp3');
+
+    // Generate a real MP3 file using ffmpeg
+    try {
+        execSync(`ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 10 -q:a 9 -acodec libmp3lame "${tempFile}"`, {
+            stdio: ['ignore', 'pipe', 'pipe']
+        });
+
+        // Upload the real media file
+        const form = new FormData();
+        form.append('file', fs.createReadStream(tempFile));
+        
+        const uploadResponse = await axios.post(baseUrl, form, {
+            headers: form.getHeaders(),
+            validateStatus: status => true,
+            timeout: 5000
+        });
+
+        // Wait a short time to ensure file is available
+        await new Promise(resolve => setTimeout(resolve, 1000));
+
+        // Clean up temp file
+        fs.rmSync(tempDir, { recursive: true, force: true });
+
+        return uploadResponse.data.url;
+    } catch (error) {
+        console.error('Error creating test file:', error);
+        throw error;
+    }
+}
+
+test.serial('should handle chunking with GCS integration when configured', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a large test file first
+    const testFileUrl = await createAndUploadTestFile();
+    const requestId = uuidv4();
+
+    // Request chunking via GET
+    const chunkResponse = await axios.get(baseUrl, {
+        params: {
+            uri: testFileUrl,
+            requestId
+        },
+        validateStatus: status => true,
+        timeout: 5000
+    });
+
+    t.is(chunkResponse.status, 200, 'Chunked request should succeed');
+    t.truthy(chunkResponse.data, 'Response should contain data');
+    t.true(Array.isArray(chunkResponse.data), 'Response should be an array');
+    t.true(chunkResponse.data.length > 0, 'Should have created at least one chunk');
+
+    // Verify each chunk exists in both Azure/Local and GCS
+    for (const chunk of chunkResponse.data) {
+        // Verify Azure/Local URL is accessible
+        const azureResponse = await axios.get(convertToLocalUrl(chunk.uri), {
+            validateStatus: status => true,
+            timeout: 5000
+        });
+        t.is(azureResponse.status, 200, `Chunk should be accessible in Azure/Local: ${chunk.uri}`);
+
+        // Verify GCS URL exists and is in correct format
+        t.truthy(chunk.gcs, 'Chunk should contain GCS URL');
+        t.true(chunk.gcs.startsWith('gs://'), 'GCS URL should use gs:// protocol');
+        
+        // Check if chunk exists in fake GCS
+        const exists = await checkGCSFile(chunk.gcs);
+        t.true(exists, `Chunk should exist in GCS: ${chunk.gcs}`);
+    }
+
+    // Clean up chunks
+    const deleteResponse = await axios.delete(`${baseUrl}?operation=delete&requestId=${requestId}`);
+    t.is(deleteResponse.status, 200, 'Delete should succeed');
+
+    // Verify all chunks are deleted from both storages
+    for (const chunk of chunkResponse.data) {
+        // Verify Azure/Local chunk is gone
+        const azureResponse = await axios.get(convertToLocalUrl(chunk.uri), {
+            validateStatus: status => true,
+            timeout: 5000
+        });
+        t.is(azureResponse.status, 404, `Chunk should not be accessible in Azure/Local after deletion: ${chunk.uri}`);
+
+        // Verify GCS chunk is gone
+        const exists = await checkGCSFile(chunk.gcs);
+        t.false(exists, `Chunk should not exist in GCS after deletion: ${chunk.gcs}`);
+    }
+});
+
+test.serial('should handle chunking errors gracefully with GCS', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file to get a valid URL format
+    const validFileUrl = await createAndUploadTestFile();
+    
+    // Test with invalid URL that matches the format of our real URLs
+    const invalidUrl = validFileUrl.replace(/[^/]+$/, 'nonexistent-file.mp3');
+    const invalidResponse = await axios.get(baseUrl, {
+        params: {
+            uri: invalidUrl,
+            requestId: uuidv4()
+        },
+        validateStatus: status => true,
+        timeout: 5000
+    });
+    
+    t.is(invalidResponse.status, 500, 'Should reject nonexistent file URL');
+    t.true(invalidResponse.data.includes('Error processing media file'), 'Should indicate error processing media file');
+
+    // Test with missing URI
+    const noUriResponse = await axios.get(baseUrl, {
+        params: {
+            requestId: uuidv4()
+        },
+        validateStatus: status => true,
+        timeout: 5000
+    });
+    
+    t.is(noUriResponse.status, 400, 'Should reject request with no URI');
+    t.is(
+        noUriResponse.data,
+        'Please pass a uri and requestId on the query string or in the request body',
+        'Should return proper error message'
+    );
+});
+
 // Legacy MediaFileChunker Tests
 test.serial('should handle file upload through legacy MediaFileChunker endpoint', async t => {
     const form = new FormData();

package/helper-apps/cortex-realtime-voice-server/README.md

@@ -4,19 +4,22 @@ To install dependencies:
 
 ```bash
 bun install
+cd client
+bun install
 ```
 
 To run:
 
+Set up your .env file with the correct Cortex API key and access to the realtime voice service.
+
 ```bash
-cd client
-bun run build
-cd ..
-bun run start
+# In the server directory
+bun run dev
 ```
 
 To run in production:
 
 ```bash
+# In the server directory
 bun run start:prod
 ```

package/helper-apps/cortex-realtime-voice-server/src/SocketServer.ts

@@ -566,10 +566,10 @@ export class SocketServer {
 
     // Parallelize memory reads
     const [memorySelf, memoryUser, memoryDirectives, memoryTopics, voiceSample] = await Promise.all([
-      readMemory(socket.data.userId, socket.data.aiName, "memorySelf", 1),
-      readMemory(socket.data.userId, socket.data.aiName, "memoryUser", 1),
-      readMemory(socket.data.userId, socket.data.aiName, "memoryDirectives", 1),
-      readMemory(socket.data.userId, socket.data.aiName, "memoryTopics", 0, 0, 10),
+      readMemory(socket.data.userId, "memorySelf", 1, 0, 0, true),
+      readMemory(socket.data.userId, "memoryUser", 1, 0, 0, true),
+      readMemory(socket.data.userId, "memoryDirectives", 1, 0, 0, true),
+      readMemory(socket.data.userId, "memoryTopics", 0, 0, 10, false),
       style(socket.data.userId, socket.data.aiName, socket.data.aiStyle, [], "")
     ]);
 

package/helper-apps/cortex-realtime-voice-server/src/cortex/memory.ts

@@ -24,8 +24,8 @@ query ManageMemory($contextId: String, $chatHistory: [MultiMessage], $aiName: St
 `
 
 const READ_MEMORY = `
-query ReadMemory($contextId: String, $aiName: String, $section: String, $priority: Int, $recentHours: Int, $numResults: Int) {
-  sys_read_memory(contextId: $contextId, aiName: $aiName, section: $section, priority: $priority, recentHours: $recentHours, numResults: $numResults) {
+query ReadMemory($contextId: String, $section: String, $priority: Int, $recentHours: Int, $numResults: Int) {
+  sys_read_memory(contextId: $contextId, section: $section, priority: $priority, recentHours: $recentHours, numResults: $numResults) {
     result
     tool
     warnings
@@ -69,20 +69,20 @@ export async function manageMemory(contextId: string,
 }
 
 export async function readMemory(contextId: string,
-                                 aiName: string,
                                  section: MemorySection,
                                  priority: number = 0,
                                  recentHours: number = 0,
-                                 numResults: number = 0
+                                 numResults: number = 0,
+                                 stripMetadata: boolean = false
 ) {
 
   const variables: CortexVariables = {
     section,
     contextId,
-    aiName,
     priority,
     recentHours,
-    numResults
+    numResults,
+    stripMetadata
   }
 
   const res = await getCortexResponse(variables, READ_MEMORY);

package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts

@@ -56,6 +56,7 @@ export type CortexVariables = {
   priority?: number;
   recentHours?: number;
   numResults?: number;
+  stripMetadata?: boolean;
 }
 
 function truncateBody(body: any): string {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.17",
+  "version": "1.3.19",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/server/plugins/azureVideoTranslatePlugin.js

@@ -40,13 +40,58 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
             return {
                 isAccessible: true,
                 contentLength,
-                durationSeconds: durationSeconds || 60
+                durationSeconds: durationSeconds || 60,
+                isAzureUrl: videoUrl.includes('.blob.core.windows.net')
             };
         } catch (error) {
             throw new Error(`Failed to access video: ${error.message}`);
         }
     }
 
+    async uploadToFileHandler(videoUrl) {
+        try {
+            // Get the file handler URL from config
+            const fileHandlerUrl = config.get("whisperMediaApiUrl");
+            if (!fileHandlerUrl) {
+                throw new Error("File handler URL is not configured");
+            }
+
+            // Start heartbeat progress updates
+            const heartbeat = setInterval(() => {
+                publishRequestProgress({
+                    requestId: this.requestId,
+                    progress: 0,
+                    info: 'Uploading and processing video...'
+                });
+            }, 5000);
+
+            try {
+                // Start the fetch request
+                const response = await axios.get(fileHandlerUrl, {
+                    params: {
+                        requestId: this.requestId,
+                        fetch: videoUrl
+                    }
+                });
+
+                if (!response.data?.url) {
+                    throw new Error("File handler did not return a valid URL");
+                }
+
+                return response.data.url;
+            } finally {
+                // Always clear the heartbeat interval
+                clearInterval(heartbeat);
+            }
+        } catch (error) {
+            logger.error(`Failed to upload video to file handler: ${error.message}`);
+            if (error.response?.data) {
+                logger.error(`Response data: ${JSON.stringify(error.response.data)}`);
+            }
+            throw new Error(`Failed to upload video to file handler: ${error.message}`);
+        }
+    }
+
     async createTranslation(params) {
         const { videoUrl, sourceLanguage, targetLanguage, voiceKind, translationId } = params;
         
@@ -131,8 +176,8 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
         if (AzureVideoTranslatePlugin.lastProcessingRate && this.videoContentLength) {
             estimatedTotalTime = this.videoContentLength / AzureVideoTranslatePlugin.lastProcessingRate;
         } else {
-            // First run: estimate based on 1x calculated video duration
-            estimatedTotalTime = (this.videoContentLength * 8) / (2.5 * 1024 * 1024);
+            // First run: estimate based on 2x calculated video duration
+            estimatedTotalTime = 2 * (this.videoContentLength * 8) / (2.5 * 1024 * 1024);
         }
 
         // eslint-disable-next-line no-constant-condition
@@ -248,7 +293,7 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
         
         try {
             const translationId = `cortex-translation-${this.requestId}`;
-            const videoUrl = requestParameters.sourcevideooraudiofilepath;
+            let videoUrl = requestParameters.sourcevideooraudiofilepath;
             const sourceLanguage = requestParameters.sourcelocale;
             const targetLanguage = requestParameters.targetlocale;
             const voiceKind = requestParameters.voicekind || 'PlatformVoice';
@@ -260,6 +305,13 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
             this.videoContentLength = videoInfo.contentLength;
             logger.debug(`Video info: ${JSON.stringify(videoInfo, null, 2)}`);
 
+            // If the video is not from Azure storage, upload it to file handler
+            if (!videoInfo.isAzureUrl) {
+                logger.debug('Video is not from Azure storage, uploading to file handler...');
+                videoUrl = await this.uploadToFileHandler(videoUrl);
+                logger.debug(`Video uploaded to file handler: ${videoUrl}`);
+            }
+
             // Create translation
             const { operationUrl } = await this.createTranslation({
                 videoUrl, sourceLanguage, targetLanguage, voiceKind, translationId