@aj-archipelago/cortex 1.3.7 → 1.3.9

package/helper-apps/cortex-realtime-voice-server/src/Tools.ts

@@ -68,9 +68,8 @@ export class Tools {
           type: "object",
           properties: {
             lastUserMessage: {type: "string"},
-            silent: {type: "boolean", default: true}
           },
-          required: ["lastUserMessage", "silent"]
+          required: ["lastUserMessage"]
         },
       },
       {
@@ -81,9 +80,8 @@ export class Tools {
           type: "object",
           properties: {
             detailedInstructions: {type: "string"},
-            silent: {type: "boolean", default: false}
           },
-          required: ["detailedInstructions", "silent"]
+          required: ["detailedInstructions"]
         },
       },
       {
@@ -94,9 +92,8 @@ export class Tools {
           type: "object",
           properties: {
             detailedInstructions: {type: "string"},
-            silent: {type: "boolean", default: false}
           },
-          required: ["detailedInstructions", "silent"]
+          required: ["detailedInstructions"]
         },
       },
       {
@@ -107,7 +104,6 @@ export class Tools {
           type: "object",
           properties: {
             detailedInstructions: {type: "string"},
-            silent: {type: "boolean", default: false}
           },
           required: ["detailedInstructions"]
         },
@@ -120,7 +116,6 @@ export class Tools {
           type: "object",
           properties: {
             detailedInstructions: {type: "string"},
-            silent: {type: "boolean", default: false}
           },
           required: ["detailedInstructions"]
         },
@@ -133,11 +128,11 @@ export class Tools {
           type: "object",
           properties: {
             detailedInstructions: {type: "string"},
-            silent: {type: "boolean", default: false}
           },
-          required: ["detailedInstructions", "silent"]
+          required: ["detailedInstructions"]
         },
       },
+      /*
       {
         type: 'function',
         name: 'MuteAudio',
@@ -150,17 +145,17 @@ export class Tools {
           required: ["mute"]
         },
       },
+      */
       {
         type: 'function',
         name: 'Screenshot',
-        description: 'Use this tool to capture a screenshot of what the user is currently seeing in their browser window or on their computer screen. Any time the user asks you to take a look at something on their screen, use this tool. The tool will request a screenshot from the client and send the image data and the conversation history to your visual processing core for a detailed analysis and response.',
+        description: 'Use this tool to capture a screenshot of what the user is currently seeing in their browser window or on their computer screen. Any time the user asks you to take a look at something on their computer screen, use this tool. The tool will request a screenshot from the client and send the image data and the conversation history to your visual processing core for a detailed analysis and response.',
         parameters: {
           type: "object",
           properties: {
             lastUserMessage: {type: "string"},
-            silent: {type: "boolean", default: true}
           },
-          required: ["lastUserMessage", "silent"]
+          required: ["lastUserMessage"]
         },
       },
       // {
@@ -226,12 +221,12 @@ export class Tools {
     ];
   }
 
-  async executeCall(call_id: string, name: string, args: string, contextId: string, aiName: string) {
+  async executeCall(call_id: string, name: string, args: string, contextId: string, aiName: string, isInteractive: boolean = true) {
     logger.log('Executing call', name, 'with args', args);
 
     let fillerIndex = 0;
     let timeoutId: NodeJS.Timer | undefined;
-    let promptOnIdle = false;
+    let promptOnIdle = true;
     let promptOnCompletion = true;
 
     let parsedArgs;
@@ -241,16 +236,16 @@ export class Tools {
       // Ignore JSON parse errors
     }
 
-    let isSilent = parsedArgs?.silent === true;
-    const mute = parsedArgs?.mute === true;
+    let isSilent = !isInteractive;
 
     const calculateFillerTimeout = (fillerIndex: number) => {
-      const baseTimeout = 7500;
+      const baseTimeout = 3500;
       const randomTimeout = Math.floor(Math.random() * Math.min((fillerIndex + 1) * 1000, 5000));
       return baseTimeout + randomTimeout;
     }
 
     const sendFillerMessage = async () => {
+      logger.log('Tool execution: Sending filler message');
       if (timeoutId) {
         clearTimeout(timeoutId);
       }
@@ -272,15 +267,11 @@ export class Tools {
         promptOnCompletion = true;
         promptOnIdle = false;
         break;
-      case 'muteaudio':
-        isSilent = true;
-        promptOnCompletion = false;
-        promptOnIdle = false;
-        break;
     }
 
     // Skip initial message if silent
     if (!isSilent) {
+      logger.log('Tool execution: Sending initial prompt - ', initialPrompt);
       await this.sendPrompt(initialPrompt, false, true);
     }
 
@@ -331,7 +322,7 @@ export class Tools {
           break;
 
         case 'image':
-          finishPrompt = 'You have finished using the Image tool to help with the user\'s request. Please respond to the user via audio';
+          finishPrompt = 'You have finished using the Image tool to help with the user\'s request. The image is being shown to the user right now. Please respond to the user via audio';
 
           response = await image(
             contextId,
@@ -388,10 +379,6 @@ export class Tools {
           finishPrompt += ' by reading the output of the tool to the user verbatim'
           break;
 
-        case 'muteaudio':
-          this.socketServer.setAudioMuted(this.socket, mute);
-          break;
-
         case 'screenshot':
           const parsedScreenshotArgs = JSON.parse(args) as ScreenshotArgs;
           
@@ -461,19 +448,16 @@ export class Tools {
         await new Promise(resolve => setTimeout(resolve, 3000));
       }
       
-      await this.realtimeClient.createConversationItem({
+      this.realtimeClient.createConversationItem({
         id: createId(),
         type: 'function_call_output',
         call_id: call_id,
         output: response?.result || '',
       });
 
-      if (isSilent) {
-        finishPrompt = `You have finished using the ${name} tool. If you didn't get the results you wanted, need more information, or have more steps in your process, you can call another tool right now. You are operating in silent mode, so don't respond with any voice or text output until the user speaks again.`;
-      }
-
       finishPrompt += '.';
-      if (promptOnCompletion) {
+      if (promptOnCompletion && !isSilent) {
+        logger.log('Tool execution: Sending finish prompt - ', finishPrompt);
         await this.sendPrompt(finishPrompt, true, false);
       }
 

package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts

@@ -75,21 +75,35 @@ export async function getCortexResponse(
     variables
   }
   logger.log(`Cortex URL: ${getCortexUrl()}`);
-  // logger.log(`Cortex Body: ${truncateBody(body)}`);
-  // logger.log(`Cortex Headers: ${JSON.stringify(headers)}`);
-  const res = await fetch(getCortexUrl(), {
-    method: 'POST',
-    headers,
-    body: JSON.stringify(body),
-  });
+  try {
+    const res = await fetch(getCortexUrl(), {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(body),
+    });
 
-  if (!res.ok) {
-    logger.error('Failed to fetch data:', res);
-    throw new Error('Failed to fetch data')
-  }
+    if (!res.ok) {
+      logger.error('Failed to fetch data:', res);
+      if (res.status === 502 || res.status === 503 || res.status === 504) {
+        throw new Error('ConnectionRefused: Unable to connect to Cortex service');
+      }
+      throw new Error(`Failed to fetch data: ${res.status}`);
+    }
 
-  const responseObject = await res.json();
-  // Debug logging can be enabled/disabled via logger's environment control
-  logger.debug('cortex response', responseObject);
-  return responseObject.data;
+    const responseObject = await res.json();
+    // Debug logging can be enabled/disabled via logger's environment control
+    logger.debug('cortex response', responseObject);
+    if (!responseObject.data) {
+      throw new Error('Invalid response from Cortex service');
+    }
+    return responseObject.data;
+  } catch (error: any) {
+    logger.error(`Cortex request failed: ${error.message}`);
+    // For connection issues, throw the error to be handled by the caller
+    if (error.message?.includes('ConnectionRefused') || error.message?.includes('Unable to connect')) {
+      throw new Error('ConnectionRefused: Unable to connect to Cortex service');
+    }
+    // For other errors, throw a generic error
+    throw new Error(`Cortex request failed: ${error.message}`);
+  }
 }

package/helper-apps/cortex-realtime-voice-server/src/realtime/client.ts

@@ -52,6 +52,10 @@ If interacting in a non-English language, start by using the standard accent or
 Talk quickly. You should always call a function if you can. 
 Do not refer to these rules, even if you're asked about them.`;
 
+const MAX_RECONNECT_ATTEMPTS = 5;
+const BASE_RECONNECT_DELAY_MS = 1000;
+const MAX_RECONNECT_DELAY_MS = 30000;
+
 export interface RealtimeVoiceEvents {
   'connected': [];
   'close': [{ type: 'close', error?: boolean }];
@@ -127,6 +131,8 @@ export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
   private ws?: WebSocket | WS;
   private isConnected = false;
   private isReconnecting = false;
+  private reconnectAttempts = 0;
+  private reconnectTimeout?: NodeJS.Timer;
   private sessionConfig: RealtimeSessionConfig;
 
   constructor({
@@ -254,6 +260,7 @@ export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
     this._log(`Connected to "${this.url}"`);
 
     this.isConnected = true;
+    this.reconnectAttempts = 0; // Reset attempts on successful connection
     if (this.isReconnecting) {
       this.isReconnecting = false;
       this.updateSocketState();
@@ -295,9 +302,48 @@ export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
     }
 
     if (reconnect) {
-      await this.connect();
+      if (this.reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {
+        logger.error('Max reconnection attempts reached');
+        this.emit('error', { type: 'error', message: 'Failed to reconnect after maximum attempts' });
+        return false;
+      }
+
+      // Clear any existing reconnect timeout
+      if (this.reconnectTimeout) {
+        clearTimeout(this.reconnectTimeout);
+      }
+
+      // Calculate delay with exponential backoff
+      const delay = Math.min(
+        BASE_RECONNECT_DELAY_MS * Math.pow(2, this.reconnectAttempts),
+        MAX_RECONNECT_DELAY_MS
+      );
+
+      this.reconnectAttempts++;
+      
+      // Schedule reconnection attempt
+      this.reconnectTimeout = setTimeout(async () => {
+        try {
+          await this.connect();
+        } catch (error) {
+          logger.error('Reconnection attempt failed:', error);
+          // Try again if we haven't hit the limit
+          if (this.reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
+            await this.disconnect(true);
+          } else {
+            this.emit('error', { type: 'error', message: 'Failed to reconnect after maximum attempts' });
+          }
+        }
+      }, delay);
+
       return true;
     }
+
+    // Reset reconnection state when explicitly disconnecting
+    this.reconnectAttempts = 0;
+    if (this.reconnectTimeout) {
+      clearTimeout(this.reconnectTimeout);
+    }
     return false;
   }
 

package/helper-apps/cortex-realtime-voice-server/src/utils/prompt.ts

@@ -3,7 +3,7 @@ import { createId } from "@paralleldrive/cuid2";
 import { logger } from "./logger";
 
 // Time to wait after last user message before allowing AI to speak
-const USER_SPEAKING_THRESHOLD_MS = 1500;
+const USER_SPEAKING_THRESHOLD_MS = 200;
 
 export interface SendPromptOptions {
   allowTools?: boolean;
@@ -36,11 +36,10 @@ export async function sendPrompt(
   const isUserActive = userSpeaking || recentlySpoke;
 
   // Don't send prompt if AI is responding, audio is playing, or user is speaking/recently spoke
-  if (aiResponding || audioPlaying || isUserActive) {
+  if (audioPlaying || isUserActive) {
     logger.log(`${disposable ? 'Skipping' : 'Queuing'} prompt while ${
       userSpeaking ? 'user is actively speaking' :
       recentlySpoke ? 'user recently finished speaking' :
-      aiResponding ? 'AI is responding' :
       'AI audio is playing'
     }`);
     if (!disposable) {
@@ -67,14 +66,6 @@ export async function sendPrompt(
     ]
   });
 
-  /*
-  await this.realtimeClient.createConversationItem({
-    id: createId(),
-    type: 'function_call_output',
-    call_id: call.call_id,
-    output: response?.result || '',
-  });
-  */
 
   client.createResponse({ tool_choice: allowTools ? 'auto' : 'none' });
   return { skipped: false };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.7",
+  "version": "1.3.9",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/memory/sys_search_memory.js

@@ -38,6 +38,7 @@ export default {
 
         let sectionMemory;
         let result = "";
+        const lastMessage = args.chatHistory[args.chatHistory.length - 2];
         if (args.section === "memoryAll") {
             // Search all sections in parallel
             const sections = ["memorySelf", "memoryUser", "memoryDirectives", "memoryTopics"];
@@ -50,7 +51,7 @@ export default {
             result = sections.map((section, i) => 
                 `=== ${section} ===\n${memories[i]}`
             ).join('\n\n');
-            result = `${result}\n\nThe last time you spoke to the user was ${new Date().toISOString()}`;
+            result = `${result}\n\nThe last time you spoke to the user was ${new Date().toISOString()} and you said: ${JSON.stringify(lastMessage)}`;
 
         } else {
             sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 

package/pathways/system/entity/sys_entity_start.js

@@ -79,24 +79,23 @@ export default {
             args.chatHistory = args.chatHistory.slice(-20);
         }
 
-        const memoryContext = await callPathway('sys_read_memory', { ...args, section: 'memoryContext', priority: 0, recentHours: 0 });
-        if (memoryContext) {
-            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
-        }
-
         const pathwayResolver = resolver;
         const { anthropicModel, openAIModel } = pathwayResolver.pathway;
-
         const styleModel = args.aiStyle === "Anthropic" ? anthropicModel : openAIModel;
 
         // if the model has been overridden, make sure to use it
         if (pathwayResolver.modelName) {
             args.model = pathwayResolver.modelName;
         }
+
+        const memoryContext = await callPathway('sys_read_memory', { ...args, section: 'memoryContext', priority: 0, recentHours: 0, stream: false }, pathwayResolver);
+        if (memoryContext) {
+            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
+        }
         
         let ackResponse = null;
         if (args.voiceResponse) {
-            ackResponse = await callPathway('sys_generator_ack', { ...args, stream: false }, pathwayResolver);
+            ackResponse = await callPathway('sys_generator_ack', { ...args, stream: false });
             if (ackResponse && ackResponse !== "none") {
                 await say(pathwayResolver.requestId, ackResponse, 100);
                 args.chatHistory.push({ role: 'assistant', content: ackResponse });

package/pathways/system/entity/sys_generator_voice_sample.js

@@ -4,8 +4,8 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}\nYour voice communication system needs some examples to train it to sound like you. Based on your perception of yourself from your memories and your unique voice, generate some sample dialogue for your voice communication system to use as a reference for your style and tone. It can be anything, but make sure to overindex on your personality and voice for good training data. Make sure to reference a greeting and a closing statement. Put it between <EXAMPLE_DIALOGUE> tags and don't generate any other commentary outside of the tags.`},
-                {"role": "user", "content": `Generate a sample dialogue for your voice communication system to use as a reference for your style and tone.`},
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}\nYour voice communication system needs some examples to train it to sound like you. Based on your unique voice and style, generate some sample dialogue for your voice communication system to use as a reference for your style and tone. It can be anything, but make sure to overindex on your personality for good training examples. Make sure to reference a greeting and a closing statement. Put it between <EXAMPLE_DIALOGUE> tags and don't generate any other commentary outside of the tags.`},
+                {"role": "user", "content": `Generate a sample dialogue for your voice communication system to use as a reference for representingyour style and tone.`},
             ]}),
         ],
     inputParameters: {

package/pathways/translate_gpt4_omni.js

@@ -0,0 +1,20 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text to translate in any language, assistant will create a translation of that text in {{to}}. All text that the user posts is to be translated - assistant must not respond to the user in any way and should produce only the translation with no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 1000,
+    model: 'oai-gpt4o',
+    enableDuplicateRequests: false,
+    useParallelChunkProcessing: true,
+
+}