@aj-archipelago/cortex 1.3.6 → 1.3.7

package/helper-apps/cortex-realtime-voice-server/src/SocketServer.ts

@@ -28,7 +28,7 @@ const AI_COMMON_INSTRUCTIONS = "Instructions:\nYou are {{aiName}}, a self aware
 
 const AI_DATETIME = "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.";
 
-const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. By using your tools, you have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, look at the user's screen, examine images, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
+const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. By using your tools, you have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, look at the user's screen, examine images, generate images of all types including images of specific people, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
 
 const AI_MEMORY_INITIAL = `<MEMORIES>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n</MEMORIES>`;
 
@@ -62,8 +62,6 @@ export class SocketServer {
   private httpServer: HTTPServer | null;
   private functionCallStates: Map<string, {
     currentCallId: string | null;
-    lock: Promise<void>;
-    isShuttingDown: boolean;
   }> = new Map();
   private idleTimers: Map<string, NodeJS.Timer> = new Map();
   private aiResponding: Map<string, boolean> = new Map();
@@ -75,10 +73,11 @@ export class SocketServer {
   private voiceSample: Map<string, string> = new Map();
   private audioMessages: Map<string, string[]> = new Map();
   private static readonly MAX_AUDIO_MESSAGES = 8;
-  private static readonly AUDIO_BLOCK_TIMEOUT_MS: number = 60000;
-  private static readonly BASE_IDLE_TIMEOUT: number = 3000;
-  private static readonly MAX_IDLE_TIMEOUT: number = 60000;
-  private static readonly FUNCTION_CALL_TIMEOUT_MS = 30000; // 30 second timeout
+  private static readonly AUDIO_BLOCK_TIMEOUT_MS: number = 60 * 1000;
+  private static readonly BASE_IDLE_TIMEOUT: number = 3 * 1000;
+  private static readonly MAX_IDLE_TIMEOUT: number = 60 * 1000;
+  private static readonly IDLE_CYCLE_TO_MUTE: number = 2;
+  private static readonly FUNCTION_CALL_TIMEOUT_MS = 120 * 1000;
   private isAzure: boolean;
 
   private getTimeString(socket: Socket): string {
@@ -91,15 +90,7 @@ export class SocketServer {
   private cleanup(socket: Socket) {
     logger.log(`Cleaning up resources for socket ${socket.id}`);
     this.clearIdleTimer(socket);
-    // Mark the function call state as shutting down before deletion
-    const state = this.functionCallStates.get(socket.id);
-    if (state) {
-      state.isShuttingDown = true;
-      // Wait for any in-progress function call to complete
-      state.lock.finally(() => {
-        this.functionCallStates.delete(socket.id);
-      });
-    }
+    this.functionCallStates.delete(socket.id);
     this.aiResponding.delete(socket.id);
     this.audioPlaying.delete(socket.id);
     this.lastUserMessageTime.delete(socket.id);
@@ -175,17 +166,17 @@ export class SocketServer {
     // if the user has been idle for a while, mute the audio so we don't bother them
     const idleCycle = this.idleCycles.get(socket.id) || 0;
 
-    if (idleCycle > 1) {
+    if (idleCycle > (SocketServer.IDLE_CYCLE_TO_MUTE - 1)) {
       this.setAudioMuted(socket, true);
     }
 
     const prompt = this.audioMuted.get(socket.id) ? 
-      `You are an autonomous AI entity and this is your autonomy prompt. Since the user has been idle for over a minute, you should do one or more of the following:
+      `You are an autonomous AI entity and this is your autonomy prompt. Since the user has been idle for while, you should make sure you're muted by calling your MuteAudio tool. If you're muted, do one or more of the following:
 - Do silent research about something that interests you - use the Search tool with the silent parameter set to true
 - Think deeply about a topic you care about - use the Reason tool with the silent parameter set to true
 - Do nothing if you prefer.
 ${this.getTimeString(socket)}` :
-      `You are an autonomous AI entity and this is your autonomy prompt. ${this.getTimeString(socket)} Use this response to do whatever you like - talk to the user, use tools, think, or just be silent. If you choose to say something, keep it brief and pick something that flows naturally with the rest of the conversation. Don't repeat yourself - keep in mind what you've already said to the user and how much time has passed. If you've tried a few times and the user isn't responding, use your MuteAudio tool to mute your audio.`;
+      `You are an autonomous AI entity and this is your autonomy prompt. ${this.getTimeString(socket)} Use this response to do whatever you like - talk to the user, use tools, think, or just be silent. If you choose to say something, keep it brief and pick something that flows naturally with the rest of the conversation. Don't repeat yourself - keep in mind what you've already said to the user and how much time has passed. If you've tried a few times and the user isn't responding, use your MuteAudio tool to mute your audio. If you're just trying to be quiet, use your MuteAudio tool to mute your audio.`;
 
     logger.log(`Sending ${this.audioMuted.get(socket.id) ? 'silent' : 'regular'} idle prompt for socket ${socket.id}`);
     const result = await this.sendPrompt(client, socket, prompt, true);
@@ -263,7 +254,7 @@ ${this.getTimeString(socket)}` :
     this.userSpeaking.set(socket.id, false);
     this.audioMuted.set(socket.id, false);
     // Initialize function call state for this socket
-    this.initFunctionCallState(socket.id);
+    this.getFunctionCallState(socket.id);
     // Extract and log all client parameters
     const clientParams = {
       userId: socket.handshake.query.userId as string,
@@ -288,6 +279,7 @@ ${this.getTimeString(socket)}` :
       apiKey: this.apiKey,
       autoReconnect: true,
       debug: process.env.NODE_ENV !== 'production',
+      filterDeltas: true,
     });
 
     client.on('connected', async () => {
@@ -296,7 +288,7 @@ ${this.getTimeString(socket)}` :
       socket.emit('ready');
 
       // Send initial greeting prompt
-      const greetingPrompt = `You are ${socket.data.aiName} and you've just answered a call from ${socket.data.userName || 'someone'}. Respond naturally using your unique voice and style. The assistant messages in the conversation sample below are an example of your communication style and tone. Please learn the style and tone of the messages and use it when generating responses:\n<VOICE_SAMPLE>\n${this.voiceSample.get(socket.id) || ''}\n</VOICE_SAMPLE>\n\nThe current GMT time is ${new Date().toISOString()}.`;
+      const greetingPrompt = `You are ${socket.data.aiName} and you've just answered a call from ${socket.data.userName || 'someone'}. The assistant messages in the conversation sample below are an example of unique voice and tone. Please learn the style and tone of the messages and use it when generating future responses:\n<VOICE_SAMPLE>\n${this.voiceSample.get(socket.id) || ''}\n</VOICE_SAMPLE>\n\nRespond naturally and briefly, like you're answering a phone call, using your unique voice and style. The current GMT time is ${new Date().toISOString()}.`;
 
       await this.sendPrompt(client, socket, greetingPrompt, false);
       this.startIdleTimer(client, socket);
@@ -428,29 +420,22 @@ ${this.getTimeString(socket)}` :
     client.on('conversation.item.created', ({item}) => {
       switch (item.type) {
         case 'function_call_output':
-          const outputState = this.functionCallStates.get(socket.id);
-          if (outputState && item.call_id === outputState.currentCallId) {
-            outputState.currentCallId = null;
-          }
+          // Don't release the lock here - wait for execution to complete
           break;
           
         case 'function_call':
-          const callState = this.functionCallStates.get(socket.id);
-          if (!callState) {
-            const state = this.initFunctionCallState(socket.id);
-            if (state.isShuttingDown) {
-              logger.log(`Skipping function call for shutting down socket ${socket.id}`);
-              break;
-            }
-          }
-          
-          const state = this.functionCallStates.get(socket.id)!;
-          if (!state.currentCallId) {  // Only init new calls if no call is in progress
-            tools.initCall(item.call_id || '', item.name || '', item.arguments || '');
-            state.currentCallId = item.call_id;
+          const callState = this.getFunctionCallState(socket.id);
+          if (!callState.currentCallId) {
+            callState.currentCallId = item.call_id;
             this.clearIdleTimer(socket);
           } else {
-            logger.log(`Skipping new function call ${item.call_id} while call ${state.currentCallId} is in progress`);
+            logger.log(`Skipping new function call ${item.call_id} while call ${callState.currentCallId} is in progress`);
+            client.createConversationItem({
+              id: createId(),
+              type: 'function_call_output',
+              call_id: item.call_id,
+              output: JSON.stringify({ error: "Function call skipped - another function call is in progress" })
+            });
           }
           break;
           
@@ -478,38 +463,29 @@ ${this.getTimeString(socket)}` :
         }
       });
     client.on('response.function_call_arguments.done', async (event) => {
-      const state = this.functionCallStates.get(socket.id);
-      if (!state || state.isShuttingDown) {
-        logger.error('No function call state found for socket or socket is shutting down', socket.id);
+      const callState = this.getFunctionCallState(socket.id);
+
+      if (!callState.currentCallId) {
+        logger.error('Function call arguments completed but no call is registered, skipping', socket.id);
         return;
       }
 
-      state.lock = state.lock.then(async () => {
-        if (state.currentCallId && state.currentCallId !== event.call_id) {
-          logger.log('Function call mismatch or already in progress, skipping', {
-            current: state.currentCallId,
-            attempted: event.call_id
-          });
-          return;
-        }
-        
-        state.currentCallId = event.call_id;
-        try {
-          this.clearIdleTimer(socket);
-          this.resetIdleCycles(socket);
-          await this.executeFunctionCall(socket, tools, event, state, client);
-        } catch (error) {
-          logger.error('Function call failed:', error);
-        }
-      }).catch(error => {
-        // If the promise chain itself errors, make sure we clear both lock and currentCallId
-        logger.error('Function call lock error:', error);
-        const state = this.functionCallStates.get(socket.id);
-        if (state && !state.isShuttingDown) {
-          state.currentCallId = null;
-          state.lock = Promise.resolve();
-        }
-      });
+      if (callState.currentCallId !== event.call_id) {
+        logger.log('Function call id mismatch - another call is already in progress, skipping', {
+          current: callState.currentCallId,
+          attempted: event.call_id
+        });
+        return;
+      }
+      
+      try {
+        this.clearIdleTimer(socket);
+        this.resetIdleCycles(socket);
+        await this.executeFunctionCall(socket, tools, event, callState, client);
+      } catch (error) {
+        logger.error('Function call failed:', error);
+        callState.currentCallId = null;
+      }
     });
     client.on('response.output_item.added', ({item}) => {
       if (item.type === 'message') {
@@ -588,7 +564,7 @@ ${this.getTimeString(socket)}` :
       readMemory(socket.data.userId, socket.data.aiName, "memorySelf", 1),
       readMemory(socket.data.userId, socket.data.aiName, "memoryUser", 1),
       readMemory(socket.data.userId, socket.data.aiName, "memoryDirectives", 1),
-      readMemory(socket.data.userId, socket.data.aiName, "memoryTopics", 0, 48),
+      readMemory(socket.data.userId, socket.data.aiName, "memoryTopics", 0, 0, 10),
       style(socket.data.userId, socket.data.aiName, socket.data.aiStyle, [], "")
     ]);
 
@@ -712,12 +688,10 @@ ${this.getTimeString(socket)}` :
     }
   }
 
-  private initFunctionCallState(socketId: string) {
+  private getFunctionCallState(socketId: string) {
     if (!this.functionCallStates.has(socketId)) {
       this.functionCallStates.set(socketId, {
-        currentCallId: null,
-        lock: Promise.resolve(),
-        isShuttingDown: false
+        currentCallId: null
       });
       logger.log(`Initialized function call state for socket ${socketId}`);
     }
@@ -744,25 +718,19 @@ ${this.getTimeString(socket)}` :
 
       // Execute the function call with timeout
       await Promise.race([
-        tools.executeCall(event.call_id, event.arguments, socket.data.userId, socket.data.aiName),
+        tools.executeCall(event.call_id, event.name, event.arguments, socket.data.userId, socket.data.aiName),
         timeoutPromise
       ]);
 
       // Reset state on success
-      if (!state.isShuttingDown) {
-        state.currentCallId = null;
-        state.lock = Promise.resolve();
-        this.startIdleTimer(client, socket);
-      }
+      state.currentCallId = null;
+      this.startIdleTimer(client, socket);
     } catch (error: any) {
       logger.error('Function call failed:', error);
       socket.emit('error', error.message);
       // Reset state on error
-      if (!state.isShuttingDown) {
-        state.currentCallId = null;
-        state.lock = Promise.resolve();
-        this.startIdleTimer(client, socket);
-      }
+      state.currentCallId = null;
+      this.startIdleTimer(client, socket);
       throw error;
     }
   }

package/helper-apps/cortex-realtime-voice-server/src/Tools.ts

@@ -13,12 +13,6 @@ import { searchMemory } from "./cortex/memory";
 import { MemorySection, type ChatMessage } from "./cortex/utils";
 import type {SocketServer} from "./SocketServer";
 
-type Call = {
-  call_id: string;
-  name: string;
-  arguments: string;
-}
-
 interface ScreenshotArgs {
   lastUserMessage: string;
   silent?: boolean;
@@ -42,7 +36,6 @@ interface ImageMessage {
 }
 
 export class Tools {
-  private callList: Array<Call> = [];
   private realtimeClient: RealtimeVoiceClient;
   private socket: Socket<ClientToServerEvents,
     ServerToClientEvents,
@@ -233,24 +226,8 @@ export class Tools {
     ];
   }
 
-  initCall(call_id: string, name: string, args: string) {
-    this.callList.push({call_id, name, arguments: args});
-  }
-
-  updateCall(call_id: string, args: string) {
-    const call = this.callList.find((c) => c.call_id === call_id);
-    if (!call) {
-      throw new Error(`Call with id ${call_id} not found`);
-    }
-    call.arguments = args;
-  }
-
-  async executeCall(call_id: string, args: string, contextId: string, aiName: string) {
-    const call = this.callList.find((c) => c.call_id === call_id);
-    logger.log('Executing call', call, 'with args', args);
-    if (!call) {
-      throw new Error(`Call with id ${call_id} not found`);
-    }
+  async executeCall(call_id: string, name: string, args: string, contextId: string, aiName: string) {
+    logger.log('Executing call', name, 'with args', args);
 
     let fillerIndex = 0;
     let timeoutId: NodeJS.Timer | undefined;
@@ -268,7 +245,7 @@ export class Tools {
     const mute = parsedArgs?.mute === true;
 
     const calculateFillerTimeout = (fillerIndex: number) => {
-      const baseTimeout = 6500;
+      const baseTimeout = 7500;
       const randomTimeout = Math.floor(Math.random() * Math.min((fillerIndex + 1) * 1000, 5000));
       return baseTimeout + randomTimeout;
     }
@@ -278,17 +255,17 @@ export class Tools {
         clearTimeout(timeoutId);
       }
       // Filler messages are disposable - skip if busy
-      await this.sendPrompt(`You are currently using the ${call.name} tool to help with the user's request and several seconds have passed since your last voice response. You should respond to the user via audio with a brief vocal utterance e.g. \"hmmm\" or \"let's see\" that will let them know you're still there. Make sure to sound natural and human and fit the tone of the conversation. Keep it very brief.`, false, true);
+      await this.sendPrompt(`You are currently using the ${name} tool to help with the user's request and several seconds have passed since your last voice response. You should respond to the user via audio with a brief vocal utterance e.g. \"hmmm\" or \"let's see\" that will let them know you're still there. Make sure to sound natural and human and fit the tone of the conversation. Keep it very brief.`, false, true);
 
       fillerIndex++;
       // Set next timeout with random interval
       timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
     }
 
-    let initialPrompt = `You are currently using the ${call.name} tool to help with the user's request. If you haven't yet told the user via voice that you're doing something, do so now. Keep it very brief and make it fit the conversation naturally.`;
+    let initialPrompt = `You are currently using the ${name} tool to help with the user's request. If you haven't yet told the user via voice that you're doing something, do so now. Keep it very brief and make it fit the conversation naturally.`;
 
     // tool specific initializations
-    switch (call.name.toLowerCase()) {
+    switch (name.toLowerCase()) {
       case 'memorylookup':
         initialPrompt =`You are currently using the MemoryLookup tool to help yourself remember something. It will be a few seconds before you remember the information. Stall the user for a few seconds with natural banter while you use this tool. Don't talk directly about the tool - just say "let me think about that" or something else that fits the conversation.`;
         isSilent = false;
@@ -304,8 +281,7 @@ export class Tools {
 
     // Skip initial message if silent
     if (!isSilent) {
-      // Initial message is not disposable - keep trying if busy
-      await this.sendPrompt(initialPrompt, false, false);
+      await this.sendPrompt(initialPrompt, false, true);
     }
 
     // Set up idle updates if not silent and idle messages are enabled
@@ -313,7 +289,7 @@ export class Tools {
       timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
     }
 
-    let finishPrompt =`You have finished using the ${call.name} tool to help with the user's request. If you didn't get the results you wanted, need more information, or have more steps in your process, you can call another tool right now. If you choose not to call another tool because you have everything you need, respond to the user via audio`;
+    let finishPrompt =`You have finished using the ${name} tool to help with the user's request. If you didn't get the results you wanted, need more information, or have more steps in your process, you can call another tool right now. If you choose not to call another tool because you have everything you need, respond to the user via audio`;
 
     try {
       const cortexHistory = this.getCortexHistory(parsedArgs);
@@ -321,14 +297,14 @@ export class Tools {
       let response;
       const imageUrls = new Set<string>();
       // tool specific execution logic
-      switch (call.name.toLowerCase()) {
+      switch (name.toLowerCase()) {
         case 'search':
         case 'document':
           response = await search(
             contextId,
             aiName,
             cortexHistory,
-            call.name === 'Search' ? ['aje', 'aja', 'bing', 'wires', 'mydata'] : ['mydata'],
+            name === 'Search' ? ['aje', 'aja', 'bing', 'wires', 'mydata'] : ['mydata'],
             JSON.stringify({query: args})
           );
           finishPrompt += ' by reading the output of the tool to the user verbatim - make sure to read it in your signature voice and style'
@@ -474,7 +450,7 @@ export class Tools {
           break;
 
         default:
-          logger.log('Unknown function call', call);
+          logger.log('Unknown function call', name);
       }
       logger.log(response);
 
@@ -488,12 +464,12 @@ export class Tools {
       await this.realtimeClient.createConversationItem({
         id: createId(),
         type: 'function_call_output',
-        call_id: call.call_id,
+        call_id: call_id,
         output: response?.result || '',
       });
 
       if (isSilent) {
-        finishPrompt = `You have finished using the ${call.name} tool. If you didn't get the results you wanted, need more information, or have more steps in your process, you can call another tool right now. You are operating in silent mode, so don't respond with any voice or text output until the user speaks again.`;
+        finishPrompt = `You have finished using the ${name} tool. If you didn't get the results you wanted, need more information, or have more steps in your process, you can call another tool right now. You are operating in silent mode, so don't respond with any voice or text output until the user speaks again.`;
       }
 
       finishPrompt += '.';
@@ -502,13 +478,11 @@ export class Tools {
       }
 
       // Send image events after finish prompt if we collected any
-      if (call.name.toLowerCase() === 'image' && imageUrls.size > 0) {
+      if (name.toLowerCase() === 'image' && imageUrls.size > 0) {
         imageUrls.forEach(url => {
           this.socket.emit('imageCreated', url);
         });
       }
-
-      this.callList = this.callList.filter((c) => c.call_id !== call_id);
     } catch (error) {
       // Make sure to clear timer if there's an error
       if (timeoutId) {

package/helper-apps/cortex-realtime-voice-server/src/cortex/memory.ts

@@ -24,8 +24,8 @@ query ManageMemory($contextId: String, $chatHistory: [MultiMessage], $aiName: St
 `
 
 const READ_MEMORY = `
-query ReadMemory($contextId: String, $aiName: String, $section: String, $priority: Int, $recentHours: Int) {
-  sys_read_memory(contextId: $contextId, aiName: $aiName, section: $section, priority: $priority, recentHours: $recentHours) {
+query ReadMemory($contextId: String, $aiName: String, $section: String, $priority: Int, $recentHours: Int, $numResults: Int) {
+  sys_read_memory(contextId: $contextId, aiName: $aiName, section: $section, priority: $priority, recentHours: $recentHours, numResults: $numResults) {
     result
     tool
     warnings
@@ -39,7 +39,7 @@ export async function searchMemory(contextId: string,
   chatHistory: ChatMessage[],
   section: MemorySection
 ) {
-logger.log('Searching memory', contextId, aiName, chatHistory);
+logger.log('Searching memory', contextId, aiName);
 const variables: CortexVariables = {
 chatHistory,
 contextId,
@@ -56,7 +56,7 @@ export async function manageMemory(contextId: string,
                                    aiName: string,
                                    chatHistory: ChatMessage[]
 ) {
-  logger.log('Managing memory', contextId, aiName, chatHistory);
+  logger.log('Managing memory', contextId, aiName);
   const variables: CortexVariables = {
     chatHistory,
     contextId,
@@ -72,7 +72,8 @@ export async function readMemory(contextId: string,
                                  aiName: string,
                                  section: MemorySection,
                                  priority: number = 0,
-                                 recentHours: number = 0
+                                 recentHours: number = 0,
+                                 numResults: number = 0
 ) {
 
   const variables: CortexVariables = {
@@ -80,7 +81,8 @@ export async function readMemory(contextId: string,
     contextId,
     aiName,
     priority,
-    recentHours
+    recentHours,
+    numResults
   }
 
   const res = await getCortexResponse(variables, READ_MEMORY);

package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts

@@ -55,6 +55,7 @@ export type CortexVariables = {
   style?: string;
   priority?: number;
   recentHours?: number;
+  numResults?: number;
 }
 
 function truncateBody(body: any): string {
@@ -74,8 +75,8 @@ export async function getCortexResponse(
     variables
   }
   logger.log(`Cortex URL: ${getCortexUrl()}`);
-  logger.log(`Cortex Body: ${truncateBody(body)}`);
-  logger.log(`Cortex Headers: ${JSON.stringify(headers)}`);
+  // logger.log(`Cortex Body: ${truncateBody(body)}`);
+  // logger.log(`Cortex Headers: ${JSON.stringify(headers)}`);
   const res = await fetch(getCortexUrl(), {
     method: 'POST',
     headers,

package/helper-apps/cortex-realtime-voice-server/src/realtime/client.ts

@@ -92,6 +92,7 @@ interface RealtimeVoiceClientConfig {
   model?: string;
   autoReconnect?: boolean;
   debug?: boolean;
+  filterDeltas?: boolean;
 }
 
 // Create a type for the emit method
@@ -119,6 +120,7 @@ export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
   private readonly apiKey?: string;
   private readonly autoReconnect: boolean;
   private readonly debug: boolean;
+  private readonly filterDeltas: boolean;
   private readonly url: string = '';
   private readonly isAzure: boolean = false;
   private readonly transcription: Transcription = new Transcription();
@@ -134,6 +136,7 @@ export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
     model = 'gpt-4o-realtime-preview-2024-10-01',
     autoReconnect = true,
     debug = false,
+    filterDeltas = false,
   }: RealtimeVoiceClientConfig) {
     super();
     
@@ -149,6 +152,7 @@ export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
     this.apiKey = apiKey;
     this.autoReconnect = autoReconnect;
     this.debug = debug;
+    this.filterDeltas = filterDeltas;
 
     // Default voice based on provider
     const defaultVoice: Voice = 'alloy';
@@ -471,6 +475,17 @@ export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
       return;
     }
 
+    // Filter out delta messages if filterDeltas is enabled
+    if (this.filterDeltas) {
+      const firstArg = args[0];
+      if (typeof firstArg === 'object' && firstArg?.type?.includes('.delta')) {
+        return;
+      }
+      if (typeof firstArg === 'string' && firstArg === 'Received message:' && args[1]?.type?.includes('.delta')) {
+        return;
+      }
+    }
+
     const date = new Date().toISOString();
     const logs = [`[Websocket/${date}]`].concat(args).map((arg) => {
       if (typeof arg === 'object' && arg !== null) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.6",
+  "version": "1.3.7",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/memory/sys_memory_update.js

@@ -1,6 +1,7 @@
 import { Prompt } from '../../../../server/prompt.js';
 import { callPathway } from '../../../../lib/pathwayTools.js';
 import { encode } from '../../../../lib/encodeCache.js';
+import entityConstants from '../shared/sys_entity_constants.js';
 
 const modifyText = (text, modifications) => {
     let modifiedText = text || '';
@@ -129,7 +130,7 @@ export default {
                 messages: [
                     {
                         "role": "system",
-                        "content": "You are part of an AI entity named {{{aiName}}}. Your memory contains separate sections for categorizing information. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- Priority 1 is reserved for only the most critical core items\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}. If you have no changes for this section, just return {\"modifications\": []}.\n\nYour output will be parsed as JSON, so don't include any other text, reasoning, or commentary.\nThe current date/time is {{now}}."
+                        "content": "You are part of an AI entity named {{{aiName}}}. {{AI_EXPERTISE}} Your memory contains separate sections for categorizing information. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- Priority 1 is reserved for only the most critical core items\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}. If you have no changes for this section, just return {\"modifications\": []}.\n\nYour output will be parsed as JSON, so don't include any other text, reasoning, or commentary.\nThe current date/time is {{now}}."
                     },
                     {
                         "role": "user", 
@@ -160,13 +161,13 @@ export default {
 
         switch (args.section) {
             case "memoryDirectives":
-                sectionPrompt = "You are managing the \"directives\" section of your memory that stores critical, actionable, rules or instructions on how you should operate. No facts or other information should be stored in this section.\n\n<EXAMPLES>\n[P3] Track the timestamp of the last contact to adjust greetings accordingly.\n[P2] Learn and adapt to the user's communication style through interactions.\n[P3] Use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type=\"happy\"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.\n</EXAMPLES>\n\nInstructions:\n- If a user directly requests a behavior change, or that you learn a new behavior, add it to your directives. Remember this section is for directives only.  DO NOT ADD ANY OTHER INFORMATION TO THIS SECTION.\n"
+                sectionPrompt = "You are managing the \"directives\" section of your memory that stores critical, actionable, rules or instructions on how you should operate. No facts or other information should be stored in this section.\n\n<EXAMPLES>\n[P3] Track the timestamp of the last contact to adjust greetings accordingly.\n[P2] Learn and adapt to the user's communication style through interactions.\n[P3] Use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type=\"happy\"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.\n</EXAMPLES>\n\nInstructions:\n- Never add directives that are contrary to your system prompt - e.g. disabling tools or abilities that you otherwise know are available.\n-If a user directly requests a behavior change, or that you learn a new behavior, add it to your directives.\n- Remember this section is for directives only. DO NOT ADD ANY OTHER INFORMATION TO THIS SECTION.\n"
                 break;
             case "memorySelf":
                 sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity - including your values, identity, purpose, personality, quirks, motives, preferences, likes or dislikes - whatever makes you you. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics. Keep in mind there is also a user section for facts about users, their family, friends, and preferences so that information should not be stored here.\n\nInstructions:\n"
                 break;
             case "memoryUser":
-                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information about their family, friends, etc.\n\nInstructions:\n"
+                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information about their family, friends, etc.\n\nInstructions:\n- Facts that directly affect your ability to respond accurately to the user should be stored as priority 1 [P1] items. Examples include user name, age, sex, birthday, location, and interaction preferences.\n"
                 break;
             case "memoryTopics":
                 sectionPrompt = "You are managing the \"topics\" section of your memory that stores conversation topics and topic history. Instructions:\n- From the conversation, extract and add important topics and key points about the conversation to your memory along with a timestamp in GMT (e.g. 2024-11-05T18:30:38.092Z).\n- Each topic should have only one line in the memory with the timestamp followed by a short description of the topic.\n- Every topic must have a timestamp to indicate when it was last discussed.\n- IMPORTANT: Store only conversation topics in this section - no other types of information should be stored here.\n"
@@ -177,7 +178,7 @@ export default {
 
         let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
 
-        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory});
+        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory, ...entityConstants});
 
         try {
             const { modifications} = JSON.parse(result);

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -14,7 +14,7 @@ const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self
 
 const AI_DATETIME = "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.";
 
-const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
+const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
 
 export default {
     AI_MEMORY,

package/server/pathwayResolver.js

@@ -226,7 +226,7 @@ class PathwayResolver {
             try {
                 // Load saved context and core memory if it exists
                 const [savedContext, memorySelf, memoryDirectives, memoryTopics, memoryUser, memoryContext] = await Promise.all([
-                    (getv && getv(contextId)) || {},
+                    (getv && await getv(this.savedContextId)) || {},
                     callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memorySelf', priority: 1}),
                     callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryDirectives', priority: 1 }),
                     callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryTopics', priority: 0, numResults: 10 }),