@aj-archipelago/cortex 1.3.18 → 1.3.20

package/README.md

@@ -1,5 +1,5 @@
 # Cortex
-Cortex simplifies and accelerates the process of creating applications that harness the power of modern AI models like GPT-4o (chatGPT), o1, Gemini, the Claude series, Flux, Grok and more by poviding a structured interface (GraphQL or REST) to a powerful prompt execution environment. This enables complex augmented prompting and abstracts away most of the complexity of managing model connections like chunking input, rate limiting, formatting output, caching, and handling errors.
+Cortex simplifies and accelerates the process of creating applications that harness the power of modern AI models like GPT-4o (chatGPT), o1, o3-mini, Gemini, the Claude series, Flux, Grok and more by poviding a structured interface (GraphQL or REST) to a powerful prompt execution environment. This enables complex augmented prompting and abstracts away most of the complexity of managing model connections like chunking input, rate limiting, formatting output, caching, and handling errors.
 ## Why build Cortex?
 Modern AI models are transformational, but a number of complexities emerge when developers start using them to deliver application-ready functions. Most models require precisely formatted, carefully engineered and sequenced prompts to produce consistent results, and the responses are typically largely unstructured text without validation or formatting. Additionally, these models are evolving rapidly, are typically costly and slow to query and implement hard request size and rate restrictions that need to be carefully navigated for optimum throughput. Cortex offers a solution to these problems and provides a simple and extensible package for interacting with NL AI models.
 
@@ -20,7 +20,7 @@ Just about anything! It's kind of an LLM swiss army knife.  Here are some ideas:
   - OpenAI models:
     - GPT-4 Omni (GPT-4o)
     - GPT-4 Omni Mini (GPT-4o-mini)
-    - O1 (including o1-mini and o1-preview) (Advanced reasoning models)
+    - O1 and O3-mini (Advanced reasoning models)
     - Most of the earlier GPT models (GPT-4, 3.5 Turbo, etc.)
   - Google models:
     - Gemini 1.5 Pro
@@ -521,7 +521,7 @@ Models are configured in the `models` section of the config. Each model can have
 
 - `OPENAI-CHAT`: For OpenAI chat models (legacy GPT-3.5)
 - `OPENAI-VISION`: For multimodal models (GPT-4o, GPT-4o-mini) supporting text, images, and other content types
-- `OPENAI-REASONING`: For O1 reasoning model with vision capabilities
+- `OPENAI-REASONING`: For O1 and O3-mini reasoning models with vision capabilities
 - `OPENAI-COMPLETION`: For OpenAI completion models
 - `OPENAI-WHISPER`: For Whisper transcription
 - `GEMINI-1.5-CHAT`: For Gemini 1.5 Pro chat models

package/config.js

@@ -207,7 +207,7 @@ var config = convict({
                 "maxReturnTokens": 100000,
                 "supportsStreaming": false
             },
-            "oai-o1-mini": {
+            "oai-o3-mini": {
                 "type": "OPENAI-REASONING",
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
@@ -215,26 +215,11 @@ var config = convict({
                     "Content-Type": "application/json"
                 },
                 "params": {
-                    "model": "o1-mini"
+                    "model": "o3-mini"
                 },
                 "requestsPerSecond": 10,
-                "maxTokenLength": 128000,
-                "maxReturnTokens": 65536,
-                "supportsStreaming": false
-            },
-            "oai-o1-preview": {
-                "type": "OPENAI-REASONING",
-                "url": "https://api.openai.com/v1/chat/completions",
-                "headers": {
-                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
-                    "Content-Type": "application/json"
-                },
-                "params": {
-                    "model": "o1-preview"
-                },
-                "requestsPerSecond": 10,
-                "maxTokenLength": 128000,
-                "maxReturnTokens": 32768,
+                "maxTokenLength": 200000,
+                "maxReturnTokens": 100000,
                 "supportsStreaming": false
             },
             "azure-bing": {

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -494,25 +494,37 @@ async function deleteGCS(blobName) {
   if (!gcs) throw new Error("Google Cloud Storage is not initialized");
 
   try {
+    const bucket = gcs.bucket(GCS_BUCKETNAME);
+    const deletedFiles = [];
+
     if (process.env.STORAGE_EMULATOR_HOST) {
       // For fake GCS server, use HTTP API directly
-      const response = await axios.delete(
-        `http://localhost:4443/storage/v1/b/${GCS_BUCKETNAME}/o/${encodeURIComponent(blobName)}`,
-        { validateStatus: status => status === 200 || status === 404 }
+      const response = await axios.get(
+        `http://localhost:4443/storage/v1/b/${GCS_BUCKETNAME}/o`,
+        { params: { prefix: blobName } }
       );
-      if (response.status === 200) {
-        console.log(`Cleaned GCS file: ${blobName}`);
-        return [blobName];
+      if (response.data.items) {
+        for (const item of response.data.items) {
+          await axios.delete(
+            `http://localhost:4443/storage/v1/b/${GCS_BUCKETNAME}/o/${encodeURIComponent(item.name)}`,
+            { validateStatus: status => status === 200 || status === 404 }
+          );
+          deletedFiles.push(item.name);
+        }
       }
-      return [];
     } else {
       // For real GCS, use the SDK
-      const bucket = gcs.bucket(GCS_BUCKETNAME);
-      const file = bucket.file(blobName);
-      await file.delete();
-      console.log(`Cleaned GCS file: ${blobName}`);
-      return [blobName];
+      const [files] = await bucket.getFiles({ prefix: blobName });
+      for (const file of files) {
+        await file.delete();
+        deletedFiles.push(file.name);
+      }
+    }
+
+    if (deletedFiles.length > 0) {
+      console.log(`Cleaned GCS files: ${deletedFiles.join(', ')}`);
     }
+    return deletedFiles;
   } catch (error) {
     if (error.code !== 404) {
       console.error(`Error in deleteGCS: ${error}`);
@@ -541,4 +553,15 @@ async function ensureGCSUpload(context, existingFile) {
   return existingFile;
 }
 
-export { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs };
+// Helper function to upload a chunk to GCS
+async function uploadChunkToGCS(chunkPath, requestId) {
+    if (!gcs) return null;
+    
+    const gcsFileName = `${requestId}/${path.basename(chunkPath)}`;
+    await gcs.bucket(GCS_BUCKETNAME).upload(chunkPath, {
+        destination: gcsFileName
+    });
+    return `gs://${GCS_BUCKETNAME}/${gcsFileName}`;
+}
+
+export { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, uploadChunkToGCS };

package/helper-apps/cortex-file-handler/constants.js

@@ -77,16 +77,23 @@ export const ACCEPTED_MIME_TYPES = {
     'audio/aac': ['.aac'],
     'audio/ogg': ['.ogg'],
     'audio/flac': ['.flac'],
+    'audio/m4a': ['.m4a'],
+    'audio/x-m4a': ['.m4a'],
+    'audio/mp3': ['.mp3'],
+    'audio/mp4': ['.mp4'],
     
     // Video types
     'video/mp4': ['.mp4'],
     'video/mpeg': ['.mpeg', '.mpg'],
+    'video/mov': ['.mov'],
     'video/quicktime': ['.mov'],
     'video/x-msvideo': ['.avi'],
     'video/x-flv': ['.flv'],
+    'video/mpg': ['.mpeg', '.mpg'],
     'video/webm': ['.webm'],
-    'video/x-ms-wmv': ['.wmv'],
-    'video/3gpp': ['.3gp']
+    'video/wmv': ['.wmv'],
+    'video/3gpp': ['.3gp'],
+    'video/m4v': ['.m4v'],
 };
 
 // Helper function to check if a mime type is accepted

package/helper-apps/cortex-file-handler/index.js

@@ -1,5 +1,5 @@
 import { downloadFile, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, AZURE_STORAGE_CONTAINER_NAME } from './blobHandler.js';
+import { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, AZURE_STORAGE_CONTAINER_NAME, uploadChunkToGCS } from './blobHandler.js';
 import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
 import { ensureEncoded, ensureFileExtension, urlExists } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
@@ -104,9 +104,7 @@ async function CortexFileHandler(context, req) {
         const azureResult = useAzure ? await deleteBlob(deleteRequestId) : await deleteFolder(deleteRequestId);
         const gcsResult = [];
         if (gcs) {
-            for (const blobName of azureResult) {
-                gcsResult.push(...await deleteGCS(blobName));
-            }
+            gcsResult.push(...await deleteGCS(deleteRequestId));
         }
         
         context.res = {
@@ -393,10 +391,21 @@ async function CortexFileHandler(context, req) {
             // sequential processing of chunks
             for (let index = 0; index < chunks.length; index++) {
                 const chunkPath = chunks[index];
-                const blobName = useAzure ? await saveFileToBlob(chunkPath, requestId) : await moveFileToPublicFolder(chunkPath, requestId);
+                let blobName;
+                let gcsUrl;
+                
+                if (useAzure) {
+                    blobName = await saveFileToBlob(chunkPath, requestId);
+                } else {
+                    blobName = await moveFileToPublicFolder(chunkPath, requestId);
+                }
+                
+                // If GCS is configured, save to GCS
+                gcsUrl = await uploadChunkToGCS(chunkPath, requestId);
+                
                 const chunkOffset = chunkOffsets[index];
-                result.push({ uri: blobName, offset: chunkOffset });
-                console.log(`Saved chunk as: ${blobName}`);
+                result.push({ uri: blobName, offset: chunkOffset, gcs: gcsUrl });
+                console.log(`Saved chunk as: ${blobName}${gcsUrl ? ` and ${gcsUrl}` : ''}`);
                 await sendProgress();
             }
 

package/helper-apps/cortex-file-handler/tests/start.test.js

@@ -5,6 +5,10 @@ import axios from 'axios';
 import FormData from 'form-data';
 import { port, publicFolder, ipAddress } from '../start.js';
 import { v4 as uuidv4 } from 'uuid';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+import { execSync } from 'child_process';
 
 // Add these helper functions at the top after imports
 const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
@@ -597,8 +601,7 @@ test.serial('should handle hash reuse with Azure storage', async t => {
     const originalUrl = upload1.data.url;
     
     // Check hash exists and returns the correct URL
-    const hashCheck1 = await axios.get(baseUrl, {
-        params: { hash: testHash, checkHash: true },
+    const hashCheck1 = await axios.get(baseUrl, { hash: testHash, checkHash: true }, {
         validateStatus: status => true
     });
     t.is(hashCheck1.status, 200, 'Hash should exist after first upload');
@@ -634,8 +637,7 @@ test.serial('should handle hash reuse with Azure storage', async t => {
     await cleanupUploadedFile(t, originalUrl);
     
     // Verify hash is now gone
-    const hashCheckAfterDelete = await axios.get(baseUrl, {
-        params: { hash: testHash, checkHash: true },
+    const hashCheckAfterDelete = await axios.get(baseUrl, { hash: testHash, checkHash: true }, {
         validateStatus: status => true
     });
     t.is(hashCheckAfterDelete.status, 404, 'Hash should be gone after file deletion');
@@ -762,6 +764,144 @@ test.serial('should handle GCS URL format and accessibility', async t => {
     await cleanupUploadedFile(t, uploadResponse.data.url);
 });
 
+// Add this helper function after other helper functions
+async function createAndUploadTestFile() {
+    // Create a temporary file path
+    const tempDir = path.join(os.tmpdir(), uuidv4());
+    fs.mkdirSync(tempDir, { recursive: true });
+    const tempFile = path.join(tempDir, 'test.mp3');
+
+    // Generate a real MP3 file using ffmpeg
+    try {
+        execSync(`ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 10 -q:a 9 -acodec libmp3lame "${tempFile}"`, {
+            stdio: ['ignore', 'pipe', 'pipe']
+        });
+
+        // Upload the real media file
+        const form = new FormData();
+        form.append('file', fs.createReadStream(tempFile));
+        
+        const uploadResponse = await axios.post(baseUrl, form, {
+            headers: form.getHeaders(),
+            validateStatus: status => true,
+            timeout: 5000
+        });
+
+        // Wait a short time to ensure file is available
+        await new Promise(resolve => setTimeout(resolve, 1000));
+
+        // Clean up temp file
+        fs.rmSync(tempDir, { recursive: true, force: true });
+
+        return uploadResponse.data.url;
+    } catch (error) {
+        console.error('Error creating test file:', error);
+        throw error;
+    }
+}
+
+test.serial('should handle chunking with GCS integration when configured', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a large test file first
+    const testFileUrl = await createAndUploadTestFile();
+    const requestId = uuidv4();
+
+    // Request chunking via GET
+    const chunkResponse = await axios.get(baseUrl, {
+        params: {
+            uri: testFileUrl,
+            requestId
+        },
+        validateStatus: status => true,
+        timeout: 5000
+    });
+
+    t.is(chunkResponse.status, 200, 'Chunked request should succeed');
+    t.truthy(chunkResponse.data, 'Response should contain data');
+    t.true(Array.isArray(chunkResponse.data), 'Response should be an array');
+    t.true(chunkResponse.data.length > 0, 'Should have created at least one chunk');
+
+    // Verify each chunk exists in both Azure/Local and GCS
+    for (const chunk of chunkResponse.data) {
+        // Verify Azure/Local URL is accessible
+        const azureResponse = await axios.get(convertToLocalUrl(chunk.uri), {
+            validateStatus: status => true,
+            timeout: 5000
+        });
+        t.is(azureResponse.status, 200, `Chunk should be accessible in Azure/Local: ${chunk.uri}`);
+
+        // Verify GCS URL exists and is in correct format
+        t.truthy(chunk.gcs, 'Chunk should contain GCS URL');
+        t.true(chunk.gcs.startsWith('gs://'), 'GCS URL should use gs:// protocol');
+        
+        // Check if chunk exists in fake GCS
+        const exists = await checkGCSFile(chunk.gcs);
+        t.true(exists, `Chunk should exist in GCS: ${chunk.gcs}`);
+    }
+
+    // Clean up chunks
+    const deleteResponse = await axios.delete(`${baseUrl}?operation=delete&requestId=${requestId}`);
+    t.is(deleteResponse.status, 200, 'Delete should succeed');
+
+    // Verify all chunks are deleted from both storages
+    for (const chunk of chunkResponse.data) {
+        // Verify Azure/Local chunk is gone
+        const azureResponse = await axios.get(convertToLocalUrl(chunk.uri), {
+            validateStatus: status => true,
+            timeout: 5000
+        });
+        t.is(azureResponse.status, 404, `Chunk should not be accessible in Azure/Local after deletion: ${chunk.uri}`);
+
+        // Verify GCS chunk is gone
+        const exists = await checkGCSFile(chunk.gcs);
+        t.false(exists, `Chunk should not exist in GCS after deletion: ${chunk.gcs}`);
+    }
+});
+
+test.serial('should handle chunking errors gracefully with GCS', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file to get a valid URL format
+    const validFileUrl = await createAndUploadTestFile();
+    
+    // Test with invalid URL that matches the format of our real URLs
+    const invalidUrl = validFileUrl.replace(/[^/]+$/, 'nonexistent-file.mp3');
+    const invalidResponse = await axios.get(baseUrl, {
+        params: {
+            uri: invalidUrl,
+            requestId: uuidv4()
+        },
+        validateStatus: status => true,
+        timeout: 5000
+    });
+    
+    t.is(invalidResponse.status, 500, 'Should reject nonexistent file URL');
+    t.true(invalidResponse.data.includes('Error processing media file'), 'Should indicate error processing media file');
+
+    // Test with missing URI
+    const noUriResponse = await axios.get(baseUrl, {
+        params: {
+            requestId: uuidv4()
+        },
+        validateStatus: status => true,
+        timeout: 5000
+    });
+    
+    t.is(noUriResponse.status, 400, 'Should reject request with no URI');
+    t.is(
+        noUriResponse.data,
+        'Please pass a uri and requestId on the query string or in the request body',
+        'Should return proper error message'
+    );
+});
+
 // Legacy MediaFileChunker Tests
 test.serial('should handle file upload through legacy MediaFileChunker endpoint', async t => {
     const form = new FormData();

package/helper-apps/cortex-realtime-voice-server/README.md

@@ -4,19 +4,22 @@ To install dependencies:
 
 ```bash
 bun install
+cd client
+bun install
 ```
 
 To run:
 
+Set up your .env file with the correct Cortex API key and access to the realtime voice service.
+
 ```bash
-cd client
-bun run build
-cd ..
-bun run start
+# In the server directory
+bun run dev
 ```
 
 To run in production:
 
 ```bash
+# In the server directory
 bun run start:prod
 ```

package/helper-apps/cortex-realtime-voice-server/src/SocketServer.ts

@@ -566,10 +566,10 @@ export class SocketServer {
 
     // Parallelize memory reads
     const [memorySelf, memoryUser, memoryDirectives, memoryTopics, voiceSample] = await Promise.all([
-      readMemory(socket.data.userId, socket.data.aiName, "memorySelf", 1),
-      readMemory(socket.data.userId, socket.data.aiName, "memoryUser", 1),
-      readMemory(socket.data.userId, socket.data.aiName, "memoryDirectives", 1),
-      readMemory(socket.data.userId, socket.data.aiName, "memoryTopics", 0, 0, 10),
+      readMemory(socket.data.userId, "memorySelf", 1, 0, 0, true),
+      readMemory(socket.data.userId, "memoryUser", 1, 0, 0, true),
+      readMemory(socket.data.userId, "memoryDirectives", 1, 0, 0, true),
+      readMemory(socket.data.userId, "memoryTopics", 0, 0, 10, false),
       style(socket.data.userId, socket.data.aiName, socket.data.aiStyle, [], "")
     ]);
 

package/helper-apps/cortex-realtime-voice-server/src/cortex/memory.ts

@@ -24,8 +24,8 @@ query ManageMemory($contextId: String, $chatHistory: [MultiMessage], $aiName: St
 `
 
 const READ_MEMORY = `
-query ReadMemory($contextId: String, $aiName: String, $section: String, $priority: Int, $recentHours: Int, $numResults: Int) {
-  sys_read_memory(contextId: $contextId, aiName: $aiName, section: $section, priority: $priority, recentHours: $recentHours, numResults: $numResults) {
+query ReadMemory($contextId: String, $section: String, $priority: Int, $recentHours: Int, $numResults: Int) {
+  sys_read_memory(contextId: $contextId, section: $section, priority: $priority, recentHours: $recentHours, numResults: $numResults) {
     result
     tool
     warnings
@@ -69,20 +69,20 @@ export async function manageMemory(contextId: string,
 }
 
 export async function readMemory(contextId: string,
-                                 aiName: string,
                                  section: MemorySection,
                                  priority: number = 0,
                                  recentHours: number = 0,
-                                 numResults: number = 0
+                                 numResults: number = 0,
+                                 stripMetadata: boolean = false
 ) {
 
   const variables: CortexVariables = {
     section,
     contextId,
-    aiName,
     priority,
     recentHours,
-    numResults
+    numResults,
+    stripMetadata
   }
 
   const res = await getCortexResponse(variables, READ_MEMORY);

package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts

@@ -56,6 +56,7 @@ export type CortexVariables = {
   priority?: number;
   recentHours?: number;
   numResults?: number;
+  stripMetadata?: boolean;
 }
 
 function truncateBody(body: any): string {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.18",
+  "version": "1.3.20",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/sys_entity_continue.js

@@ -57,7 +57,15 @@ export default {
             
             const result = await callPathway(generatorPathway, newArgs, resolver);
 
-            return args.stream ? "" : result;
+            if (args.stream) {
+                return "";
+            }
+
+            if (!result) {
+                result = await callPathway('sys_generator_error', { ...args, text: `Tried to use a tool (${generatorPathway}), but no result was returned`, stream: false }, resolver);
+            }
+
+            return result;
 
         } catch (e) {
             resolver.logError(e.message ?? e);

package/pathways/system/entity/sys_generator_reasoning.js

@@ -15,7 +15,7 @@ export default {
         aiName: "Jarvis",
         language: "English",
     },
-    model: 'oai-o1',
+    model: 'oai-o3-mini',
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,

package/pathways/system/rest_streaming/{sys_openai_chat_o1_mini.js → sys_openai_chat_o3_mini.js}

@@ -1,4 +1,4 @@
-// sys_openai_chat_o1_mini.js
+// sys_openai_chat_o3_mini.js
 
 import { Prompt } from '../../../server/prompt.js';
 
@@ -12,8 +12,8 @@ export default {
     inputParameters: {
         messages: [],
     },
-    model: 'oai-o1-mini',
+    model: 'oai-o3-mini',
     useInputChunking: false,
-    emulateOpenAIChatModel: 'o1-mini',
+    emulateOpenAIChatModel: 'o3-mini',
     enableDuplicateRequests: false,
 }

package/server/plugins/azureVideoTranslatePlugin.js

@@ -56,19 +56,33 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
                 throw new Error("File handler URL is not configured");
             }
 
-            // Use the file handler's fetch endpoint
-            const response = await axios.get(fileHandlerUrl, {
-                params: {
+            // Start heartbeat progress updates
+            const heartbeat = setInterval(() => {
+                publishRequestProgress({
                     requestId: this.requestId,
-                    fetch: videoUrl
+                    progress: 0,
+                    info: 'Uploading and processing video...'
+                });
+            }, 5000);
+
+            try {
+                // Start the fetch request
+                const response = await axios.get(fileHandlerUrl, {
+                    params: {
+                        requestId: this.requestId,
+                        fetch: videoUrl
+                    }
+                });
+
+                if (!response.data?.url) {
+                    throw new Error("File handler did not return a valid URL");
                 }
-            });
 
-            if (!response.data?.url) {
-                throw new Error("File handler did not return a valid URL");
+                return response.data.url;
+            } finally {
+                // Always clear the heartbeat interval
+                clearInterval(heartbeat);
             }
-
-            return response.data.url;
         } catch (error) {
             logger.error(`Failed to upload video to file handler: ${error.message}`);
             if (error.response?.data) {