@aj-archipelago/cortex 1.3.1 → 1.3.2

package/helper-apps/cortex-autogen/agents.py

@@ -16,17 +16,32 @@ from agents_extra import process_helper_results
 from config import prompts
 import queue
 import threading
+import shutil
 
-shared_queues = {}
+human_input_queues = {}
 def background_human_input_check(request_id):
     while True:
         human_input = check_for_human_input(request_id)
         if human_input:
-            shared_queues[request_id].put(human_input)
+            human_input_queues[request_id].put(human_input)
             if human_input in ["TERMINATE", "PAUSE"]:
                 break
         time.sleep(1)
 
+def get_request_temp_dir(request_id):
+    if not request_id:
+        logging.warning("No request_id provided!")
+        return None
+    temp_dir_name = f"cortex_autogen/{request_id}"
+    temp_dir = os.path.join(tempfile.gettempdir(), temp_dir_name)
+    
+    try:
+        os.makedirs(temp_dir, exist_ok=True)
+    except OSError as e:
+        logging.error(f"Error creating temporary directory: {e}")
+        return None
+    
+    return temp_dir
 
 def find_code_message(all_messages):
     if not all_messages or len(all_messages) < 2:
@@ -101,7 +116,10 @@ def chat_with_agents(**kwargs):
         logging.warning("No all_messages list provided!")
         all_messages = []
 
-    with tempfile.TemporaryDirectory() as temp_dir:
+    # with tempfile.TemporaryDirectory() as temp_dir:
+    if True:
+        #mark the temp_dir for later upload
+        temp_dir = get_request_temp_dir(request_id)
         code_executor = autogen.coding.LocalCommandLineCodeExecutor(work_dir=temp_dir,timeout=300)
 
         assistant = AssistantAgent("assistant", llm_config=llm_config, system_message=prompt, is_termination_msg=is_termination_msg)
@@ -139,7 +157,6 @@ def chat_with_agents(**kwargs):
                 "content":corrector_result, #code_msg,
                 "task": original_request_message,
                 "contextId": original_request_message_data.get("contextId"),
-                "requestId": request_id,
             })
         except Exception as e:
             logging.error(f"Error extracting code corrector result: {e}")
@@ -180,8 +197,8 @@ def logged_send(sender, original_send, message, recipient, request_reply=None, s
         logging.log(logging.INFO, message)
     
 
-    if request_id in shared_queues and not shared_queues[request_id].empty():
-        human_input = shared_queues[request_id].get()
+    if request_id in human_input_queues and not human_input_queues[request_id].empty():
+        human_input = human_input_queues[request_id].get()
         if human_input:
             if human_input == "TERMINATE":
                 logging.info("Terminating conversation")
@@ -212,7 +229,7 @@ def process_message(original_request_message_data, original_request_message_data
         request_id = original_request_message_data.get('requestId') or original_request_message_data.id
         original_request_message = original_request_message_data['message']
 
-        shared_queues[request_id] = queue.Queue()
+        human_input_queues[request_id] = queue.Queue()
         thread = threading.Thread(target=background_human_input_check, args=(request_id,))
         thread.daemon = True
         thread.start()
@@ -229,6 +246,7 @@ def process_message(original_request_message_data, original_request_message_data
             "createdAt": datetime.now(timezone.utc).isoformat(),
             "insertionTime": original_request_message_data_obj.insertion_time.astimezone(timezone.utc).isoformat(),
             "startedAt": started_at.astimezone(timezone.utc).isoformat(),
+            "tool": "{\"toolUsed\":\"coding\"}"
         }       
 
         publish_request_progress(finalData)
@@ -256,10 +274,23 @@ def process_message(original_request_message_data, original_request_message_data
                     "createdAt": datetime.now(timezone.utc).isoformat(),
                     "insertionTime": original_request_message_data_obj.insertion_time.astimezone(timezone.utc).isoformat(),
                     "startedAt": started_at.astimezone(timezone.utc).isoformat(),
+                    "tool": "{\"toolUsed\":\"coding\"}"
                 })
         except Exception as e:
             logging.error(f"Error processing message finish publish&store: {str(e)}")
-
+    finally:
+        try:
+            #clean up the temp folder
+            temp_dir = get_request_temp_dir(request_id)
+            if temp_dir:
+                #validate cortex_autogen folder in temp_dir path
+                if "/cortex_autogen/" in temp_dir:
+                    shutil.rmtree(temp_dir)
+                else:
+                    logging.warning(f"Invalid temp_dir path: {temp_dir}, not deleting")
+        except Exception as e:
+            logging.error(f"Error cleaning up: {str(e)}")
+            
 
 
 def process_message_safe(original_request_message_data, original_request_message_data_obj, original_request_message,  all_messages, request_id, started_at):
@@ -382,11 +413,11 @@ Reply to it with task result, do not forget that user expects you continue origi
     final_msg = presenter_result
 
 
-
-    zip_url = None  # TODO: Implement if needed
-    if zip_url:
+    zip_url = zip_and_upload_tmp_folder(get_request_temp_dir(request_id))
+    if zip_url and len(zip_url) > 0:
         final_msg += f"\n\n[Download all files of this task]({zip_url})"
     
-    print(f"Task completed, task: {original_request_message}, result: {final_msg}")
+    
+    print(f"Task completed, task:\n{original_request_message},\nresult: {final_msg}")
     logging.info(f"Task completed, task:\n{original_request_message},\nresult: {final_msg}")
     return final_msg

package/helper-apps/cortex-autogen/function_app.py

@@ -7,14 +7,19 @@ import redis
 from agents import process_message
 import subprocess
 import sys
-import config 
+import config
+import requests
 
 logging.getLogger().setLevel(logging.WARNING)
 
-def install_packages():
-    subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"])
+import subprocess, sys, importlib
+required_packages = ['requests', 'azure-storage-blob']  # Add any and all other required packages
+for package in required_packages:
+    try:
+        importlib.import_module(package)
+    except ImportError:
+        subprocess.check_call([sys.executable, "-m", "pip", "install", package, "--disable-pip-version-check"], stderr=subprocess.STDOUT, stdout=subprocess.DEVNULL)
 
-# install_packages()
 
 app = func.FunctionApp()
 

package/helper-apps/cortex-autogen/search.py

@@ -73,6 +73,8 @@ def index_message(message):
         "date": datetime.now(timezone.utc).isoformat(),
         "content": message.get("content"),
         "task": message.get("task"),
+        "owner": message.get("contextId"),
+        "requestId": message.get("requestId")
     }
 
     try:

package/helper-apps/cortex-autogen/utils.py

@@ -26,13 +26,23 @@ def fetch_from_url(url):
     except requests.RequestException as e:
         logging.error(f"Error fetching from URL: {e}")
         return ""
-
+    
 def zip_and_upload_tmp_folder(temp_dir):
+    # Check if no files in temp_dir
+    if not os.listdir(temp_dir) or len(os.listdir(temp_dir)) == 0:
+        logging.info(f"No files in {temp_dir}")
+        return ""
+
     zip_path = os.path.join(temp_dir, "tmp_contents.zip")
     with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
         for root, _, files in os.walk(temp_dir):
             for file in files:
                 file_path = os.path.join(root, file)
+
+                # Skip adding the zip file itself to the archive
+                if file_path == zip_path:
+                    continue
+
                 arcname = os.path.relpath(file_path, temp_dir)
                 zipf.write(file_path, arcname)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.1",
+  "version": "1.3.2",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/chat_title.js

@@ -22,4 +22,5 @@ export default {
     model: 'oai-gpt4o',
     useInputChunking: false,
     temperature: 0,
+    enableDuplicateRequests: false
 };

package/pathways/system/entity/memory/sys_memory_required.js

@@ -4,7 +4,7 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\nYour directives and learned behaviors are:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, so look for:\n1. Personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to the AI\n\n4. Anything the user has asked you to remember or forget\n\nIf you decide to use memory, you must produce a JSON object that communicates your decision.\nReturn your decision as a JSON object like the following: {"memoryRequired": true, "memoryReason": "why you think memory is required"}. If you decide not to use memory, simply return {"memoryRequired": false}. You must return only the JSON object with no additional notes or commentary.`},
+                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\nYour directives and learned behaviors are:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, but if the conversation turn contains any of these things, you should use memory:\n1. Important personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to you to learn\n4. Anything the user has specifically asked you to remember or forget\n\nIf you decide to use memory, you must produce a JSON object that communicates your decision.\nReturn your decision as a JSON object like the following: {"memoryRequired": true, "memoryReason": "why you think memory is required"}. If you decide not to use memory, simply return {"memoryRequired": false}. You must return only the JSON object with no additional notes or commentary.`},
                 {"role": "user", "content": "Generate a JSON object to indicate if memory is required for the last turn of the conversation."},
             ]}),
         ],

package/pathways/system/entity/memory/sys_memory_update.js

@@ -33,7 +33,7 @@ const modifyText = (text, modifications) => {
     return modifiedText;
 };
 
-const enforceTokenLimit = (text, maxTokens = 15000, isTopicsSection = false) => {
+const enforceTokenLimit = (text, maxTokens = 5000, isTopicsSection = false) => {
     if (!text) return text;
     
     const lines = text.split('\n')
@@ -99,7 +99,7 @@ export default {
                 messages: [
                     {
                         "role": "system",
-                        "content": "You are part of an AI entity named {{{aiName}}}. Your memory contains separate sections for categorizing information about directives, self, user, and topics. You must keep relevant information in the appropriate section so there is no overlap or confusion. {{{sectionPrompt}}}\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. If you have no changes, just return an empty array in 'modifications'. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}\n\nYour output will be parsed as JSON, so don't include any other text or commentary.\nThe current date/time is {{now}}."
+                        "content": "You are part of an AI entity named {{{aiName}}}. Your memory contains separate sections for categorizing information about directives, self, user, and topics. You must keep relevant information in the appropriate section so there is no overlap or confusion. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. If you have no changes, just return an empty array in 'modifications'. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}\n\nYour output will be parsed as JSON, so don't include any other text or commentary.\nThe current date/time is {{now}}."
                     },
                     {
                         "role": "user", 
@@ -148,8 +148,8 @@ Follow these guidelines:
    - Specific enough for effective application
 
 3. Be selective:
-   - Store only important, actionable directives and behaviors
-   - Delete trivial or repetitive instructions
+   - Store only critical, actionable directives and behaviors
+   - Delete trivial directives or repetitive directives
 
 4. Avoid duplicates:
    - Do not add duplicate directives
@@ -185,7 +185,7 @@ Follow these guidelines:
             const { modifications} = JSON.parse(result);
             if (modifications.length > 0) {
                 sectionMemory = modifyText(sectionMemory, modifications);
-                sectionMemory = enforceTokenLimit(sectionMemory, 15000, args.section === 'memoryTopics');
+                sectionMemory = enforceTokenLimit(sectionMemory, 5000, args.section === 'memoryTopics');
                 await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
             }
             return sectionMemory;

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -8,11 +8,13 @@ const AI_CONVERSATION_HISTORY = "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}
 
 const AI_COMMON_INSTRUCTIONS = "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}";
 
-const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.\nYou know the current date and time - it is {{now}}.";
+const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.";
 
-const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice, so keep your responses very brief and conversational unless you have been explicitly asked for details. Your responses should sound like natural human conversation.\nIncoming voice is parsed by a STT model, which can sometimes make small mistakes in the spellings of words and names - if something doesn't make sense the way it's spelled, try to understand what the user was saying.\nYour voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\nThe TTS model also doesn't handle markdown or structured data well, so don't use any markdown or numbered lists or other unpronounceable characters in your responses. Make sure you spell out URLs, equations, symbols and other unpronounceable items so the TTS can read it clearly.\nYour responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.\nYou know the current date and time - it is {{now}}.";
+const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, replace the original words with a IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
 
-const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data andthe ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
+const AI_DATETIME = "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.";
+
+const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
 
 export default {
     AI_MEMORY,
@@ -21,6 +23,7 @@ export default {
     AI_COMMON_INSTRUCTIONS_MARKDOWN,
     AI_COMMON_INSTRUCTIONS_VOICE,
     AI_CONVERSATION_HISTORY,
+    AI_DATETIME,
     AI_EXPERTISE,
     AI_MEMORY_INSTRUCTIONS
 };

package/pathways/system/entity/sys_entity_start.js

@@ -40,7 +40,7 @@ export default {
     model: 'oai-gpt4o',
     anthropicModel: 'claude-35-sonnet-vertex',
     openAIModel: 'oai-gpt4o',
-    useSingleTokenStream: true,
+    useSingleTokenStream: false,
     inputParameters: {
         privateData: false,    
         chatHistory: [{role: '', content: []}],
@@ -88,9 +88,18 @@ export default {
             args.model = pathwayResolver.modelName;
         }
         
+        let ackResponse = null;
+        if (args.voiceResponse) {
+            ackResponse = await callPathway('sys_generator_ack', { ...args, stream: false }, pathwayResolver);
+            if (ackResponse) {
+                await say(pathwayResolver.requestId, ackResponse, 10);
+                args.chatHistory.push({ role: 'assistant', content: ackResponse });
+            }
+        }
+
         const fetchChatResponse = async (args, pathwayResolver) => {
             const [chatResponse, chatTitleResponse] = await Promise.all([
-                callPathway('sys_generator_quick', {...args, model: styleModel}, pathwayResolver),
+                callPathway('sys_generator_quick', {...args, model: styleModel }, pathwayResolver),
                 callPathway('chat_title', { ...args, stream: false}),
             ]);
 
@@ -104,7 +113,7 @@ export default {
         // start fetching the default response - we may need it later
         let fetchChatResponsePromise;
         if (!args.stream) {
-            fetchChatResponsePromise = fetchChatResponse({ ...args }, pathwayResolver);
+            fetchChatResponsePromise = fetchChatResponse({ ...args, ackResponse }, pathwayResolver);
         }
 
         const visionContentPresent = chatArgsHasImageUrl(args);
@@ -204,7 +213,9 @@ export default {
 
             if (toolCallbackMessage) {
                 if (args.stream) {
-                    await say(pathwayResolver.requestId, toolCallbackMessage || "One moment please.", 10);
+                    if (!ackResponse) {
+                        await say(pathwayResolver.requestId, toolCallbackMessage || "One moment please.", 10);
+                    }
                     pathwayResolver.tool = JSON.stringify({ hideFromModel: false, search: false, title });  
                     await callPathway('sys_entity_continue', { ...args, stream: true, model: styleModel, generatorPathway: toolCallbackName }, pathwayResolver);
                     return "";
@@ -222,15 +233,13 @@ export default {
                 }
             }
 
-            fetchChatResponsePromise = fetchChatResponsePromise || fetchChatResponse({ ...args }, pathwayResolver);
-            const chatResponse = await fetchChatResponsePromise;
-            pathwayResolver.tool = JSON.stringify({ search: false, title })
+            const chatResponse = await (fetchChatResponsePromise || fetchChatResponse({ ...args, ackResponse }, pathwayResolver));
+            pathwayResolver.tool = JSON.stringify({ search: false, title });
             return args.stream ? "" : chatResponse;
 
         } catch (e) {
             pathwayResolver.logError(e);
-            fetchChatResponsePromise = fetchChatResponsePromise || fetchChatResponse({ ...args }, pathwayResolver);
-            const chatResponse = await fetchChatResponsePromise;
+            const chatResponse = await (fetchChatResponsePromise || fetchChatResponse({ ...args, ackResponse }, pathwayResolver));
             pathwayResolver.tool = JSON.stringify({ search: false, title });
             return args.stream ? "" : chatResponse;
         }

package/pathways/system/entity/sys_generator_ack.js

@@ -0,0 +1,20 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\nYou are a part of an AI system named {{aiName}}. Your job is to acknowledge the user's request and provide a very brief voice filler response that is conversational and natural. The purpose of the response is just to let the user know that you have heard them and are processing a response.\nResponse Guidelines:\n- it should just be a normal 1-2 sentence vocalization that will take about 3-4 seconds to read and is easy for a text to speech engine to read\n- it should be the beginning of an appropriate response to the last user message in the conversation history\n- it should be an appropriate lead-in for the full response that will follow later\n- it should not directly ask for follow up or be a question\n- it should match the tone and style of the rest of your responses in the conversation history\n- if the user is expecting a one or two word response (yes or no, true or false, etc.) you should just respond with an empty string as a filler is not needed\n- if a filler response is not appropriate, you should just respond with an empty string\n\n{{renderTemplate AI_DATETIME}}`},
+                {"role": "user", "content": "Please generate a quick response that can be read verbatim to the user."}
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+        model: "oai-gpt4o-mini",
+    },
+    useInputChunking: false,
+    enableDuplicateRequests: false
+}

package/pathways/system/entity/sys_generator_error.js

@@ -4,7 +4,7 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_DIRECTIVES}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\nThe user has requested information that you have already determined can be found in the indexes that you can search, and you were trying to search for it, but encountered the following error: {{{text}}}. Your response should be concise, fit the rest of the conversation, include detail appropriate for the technical level of the user if you can determine it, and be appropriate for the context. You cannot resolve this error.`},
+                {"role": "system", "content": `{{renderTemplate AI_DIRECTIVES}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\nThe user has requested information that you have already determined can be found in the indexes that you can search, and you were trying to search for it, but encountered the following error: {{{text}}}. Your response should be concise, fit the rest of the conversation, include detail appropriate for the technical level of the user if you can determine it, and be appropriate for the context. You cannot resolve this error.\n{{renderTemplate AI_DATETIME}}`},
                 "{{chatHistory}}",
             ]}),
         ],

package/pathways/system/entity/sys_generator_expert.js

@@ -1,10 +1,11 @@
 import { Prompt } from '../../../server/prompt.js';
+import { callPathway } from '../../../lib/pathwayTools.js';
 
 export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}`},
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\n{{renderTemplate AI_DATETIME}}`},
                 "{{chatHistory}}",
             ]}),
         ],
@@ -19,7 +20,13 @@ export default {
     enableDuplicateRequests: false,
     timeout: 600,
     executePathway: async ({args, runAllPrompts, resolver}) => {
-        const result = await runAllPrompts({ ...args });
+        let result;
+        if (args.voiceResponse) {
+            result = await runAllPrompts({ ...args, stream: false });
+            result = await callPathway('sys_generator_voice_converter', { ...args, text: result, stream: false });
+        } else {
+            result = await runAllPrompts({ ...args });
+        }
         resolver.tool = JSON.stringify({ toolUsed: "writing" });
         return result;
     }

package/pathways/system/entity/sys_generator_image.js

@@ -45,7 +45,7 @@ export default {
 
 {{renderTemplate AI_DIRECTIVES}}
 
-Instructions: As part of a conversation with the user, you have been asked to create one or more images, photos, pictures, selfies, drawings, or other visual content for the user. You have already written the prompts and created the images - links to them are in the most recent tool calls in the chat history. You should display the images in a way that is most pleasing to the user. You can use markdown or HTML and img tags to display and format the images - the UI will render either. If there are no tool results, it means you didn't successfully create any images - in that case, don't show any images and tell the user you weren't able to create images.`
+Instructions: As part of a conversation with the user, you have been asked to create one or more images, photos, pictures, selfies, drawings, or other visual content for the user. You have already written the prompts and created the images - links to them are in the most recent tool calls in the chat history. You should display the images in a way that is most pleasing to the user. You can use markdown or HTML and img tags to display and format the images - the UI will render either. If there are no tool results, it means you didn't successfully create any images - in that case, don't show any images and tell the user you weren't able to create images.\n{{renderTemplate AI_DATETIME}}`
                 },
                 "{{chatHistory}}",
             ]}),

package/pathways/system/entity/sys_generator_quick.js

@@ -1,9 +1,10 @@
 import { Prompt } from '../../../server/prompt.js';
+
 export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\nThe UI also has dedicated tabs to help with document translation (translate), article writing assistance including generating headlines, summaries and doing copy editing (write), video and audio transcription (transcribe), and programming and writing code (code). If the user asks about something related to a dedicated tab, you will tell them that the tab exists and the interface will give the user the option to swap to that tab.\n{{renderTemplate AI_EXPERTISE}}\nYou have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation.\nNever pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}`},
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\nYou have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation.\nNever pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{#if ackResponse}}You may see short filler phrases in your past responses. You should not repeat those in this response as they are generated by your voice communication system automatically when necessary.\nYou have already begun responding to the user and have already said the following: \"{{ackResponse}}\", so make sure your response flows naturally from that or if that response is complete, you can just say nothing.\n{{/if}}{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
                 "{{chatHistory}}",
             ]}),
         ],
@@ -15,5 +16,5 @@ export default {
         model: "oai-gpt4o",
     },
     useInputChunking: false,
-    enableDuplicateRequests: false
+    enableDuplicateRequests: false,
 }

package/pathways/system/entity/sys_generator_reasoning.js

@@ -1,10 +1,11 @@
 import { Prompt } from '../../../server/prompt.js';
+import { callPathway, say } from '../../../lib/pathwayTools.js';
 
 export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nUse all of the information in your memory and the chat history to reason about the user's request and provide a response. Often this information will be more current than your knowledge cutoff.`},
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_MEMORY}}\nYou are the AI subsystem responsible for advanced, step-by-step reasoning. Use all of the information in your memory and the chat history to reason about the user's request and provide a correct and accurate response. The information in your chat history may be more current than your knowledge cutoff and has been verified by other subsystems so prioritize it over your internal knowledge.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
                 "{{chatHistory}}",
             ]}),
         ],
@@ -19,9 +20,59 @@ export default {
     enableDuplicateRequests: false,
     timeout: 600,
     executePathway: async ({args, runAllPrompts, resolver}) => {
-        const result = await runAllPrompts({ ...args, stream: false });
-        resolver.tool = JSON.stringify({ toolUsed: "reasoning" });
-        return result;
-    }
+        let timeoutId;
+
+        let fillerResponses = [];
+        if (args.voiceResponse) {
+            const voiceFillerStrings = await callPathway('sys_generator_voice_filler', { ...args, stream: false });
+            try {
+            fillerResponses = JSON.parse(voiceFillerStrings);
+            } catch (e) {
+                console.error("Error parsing voice filler responses", e);
+            }
+            if (fillerResponses.length === 0) {
+                fillerResponses = ["Please wait a moment...", "I'm working on it...", "Just a bit longer..."];
+            }
+        }
         
+        let fillerIndex = 0;
+
+        const calculateFillerTimeout = (fillerIndex) => {
+            const baseTimeout = 6500;
+            const randomTimeout = Math.floor(Math.random() * Math.min((fillerIndex + 1) * 1000, 5000));
+            return baseTimeout + randomTimeout;
+        }
+
+        const sendFillerMessage = async () => {
+            if (args.voiceResponse && Array.isArray(fillerResponses) && fillerResponses.length > 0) {
+                const message = fillerResponses[fillerIndex % fillerResponses.length];
+                await say(resolver.rootRequestId, message, 1);
+                fillerIndex++;
+                // Set next timeout with random interval
+                timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
+            }
+        };
+
+        try {
+            // Start the first timeout
+            timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
+
+            let result = await runAllPrompts({ ...args, stream: false });
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+            }
+            
+            if (args.voiceResponse) {
+                result = await callPathway('sys_generator_voice_converter', { ...args, text: result, stream: false });
+            }
+            
+            resolver.tool = JSON.stringify({ toolUsed: "reasoning" });          
+            return result;
+        } finally {
+            // Clean up timeout when done
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+            }
+        }
+    }
 }

package/pathways/system/entity/sys_generator_results.js

@@ -55,7 +55,8 @@ You can share any information you have, including personal details, addresses, o
 Here are the search strings used to find the information sources:
 <SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>\n
 Here are the information sources that were found:
-<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n`,
+<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n\n
+{{renderTemplate AI_DATETIME}}`,
                 },
                 {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request{{^if voiceResponse}} citing the sources where relevant{{/if}}. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer. Don't start the response with an affirmative like \"Sure\" or \"Certainly\". {{#if voiceResponse}}Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text only.{{/if}}"},
             ]}),
@@ -75,13 +76,49 @@ Here are the information sources that were found:
             );
         }
 
+        let timeoutId;
+
+        // Convert chatHistory to single content for rest of the code
+        const multiModalChatHistory = JSON.parse(JSON.stringify(chatHistory));
+        convertToSingleContentChatHistory(chatHistory);       
+
+        // figure out what the user wants us to do
+        const contextInfo = args.chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+
+        let fillerResponses = [];
+        if (args.voiceResponse) {
+            const voiceFillerStrings = await callPathway('sys_generator_voice_filler', { ...args, contextInfo, stream: false });
+            try {
+            fillerResponses = JSON.parse(voiceFillerStrings);
+            } catch (e) {
+                console.error("Error parsing voice filler responses", e);
+            }
+            if (fillerResponses.length === 0) {
+                fillerResponses = ["Please wait a moment...", "I'm working on it...", "Just a bit longer..."];
+            }
+        }
+
+        let fillerIndex = 0;
+
+        const calculateFillerTimeout = (fillerIndex) => {
+            const baseTimeout = 6500;
+            const randomTimeout = Math.floor(Math.random() * Math.min((fillerIndex + 1) * 1000, 5000));
+            return baseTimeout + randomTimeout;
+        }
+
+        const sendFillerMessage = async () => {
+            if (args.voiceResponse && Array.isArray(fillerResponses) && fillerResponses.length > 0) {
+                const message = fillerResponses[fillerIndex % fillerResponses.length];
+                await say(resolver.rootRequestId, message, 1);
+                fillerIndex++;
+                // Set next timeout with random interval
+                timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
+            }
+        };
+
         try {
-            // Convert chatHistory to single content for rest of the code
-            const multiModalChatHistory = JSON.parse(JSON.stringify(chatHistory));
-            convertToSingleContentChatHistory(chatHistory);
-          
-            // figure out what the user wants us to do
-            const contextInfo = chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+            // Start the first timeout
+            timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
             
             // execute the router and default response in parallel
             const [helper] = await Promise.all([
@@ -90,7 +127,7 @@ Here are the information sources that were found:
 
             logger.debug(`Search helper response: ${helper}`);
             const parsedHelper = JSON.parse(helper);
-            const { searchAJA, searchAJE, searchWires, searchPersonal, searchBing, dateFilter, languageStr, titleOnly, resultsMessage } = parsedHelper;
+            const { searchAJA, searchAJE, searchWires, searchPersonal, searchBing, dateFilter, languageStr, titleOnly } = parsedHelper;
 
             // calculate whether we have room to do RAG in the current conversation context
             const baseSystemPrompt = pathwayResolver?.prompts[0]?.messages[0]?.content;
@@ -289,8 +326,17 @@ Here are the information sources that were found:
             let sources = searchResults.map(getSource).join(" \n\n ") || "No relevant sources found.";
             dateFilter && sources.trim() && (sources+=`\n\nThe above sources are date filtered accordingly.`);
 
-            await say(pathwayResolver.rootRequestId, resultsMessage || "Let me look through these results.", 10);
-            const result = await runAllPrompts({ ...args, searchStrings: `${helper}`, sources, chatHistory: multiModalChatHistory, language:languageStr });
+            let result;
+
+            result = await runAllPrompts({ ...args, searchStrings: `${helper}`, sources, chatHistory: multiModalChatHistory, language:languageStr, stream: false });
+            
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+            }
+
+            if (args.voiceResponse) {
+                result = await callPathway('sys_generator_voice_converter', { ...args, text: result, stream: false });
+            }
 
             if (!args.stream) {
                 const referencedSources = extractReferencedSources(result);
@@ -302,9 +348,13 @@ Here are the information sources that were found:
 
             return result;
         } catch (e) {
-            //pathwayResolver.logError(e);
             const result = await callPathway('sys_generator_error', { ...args, text: JSON.stringify(e), stream: false });
-            return args.stream ? "" : result;
+            return result;
+        } finally {
+            // Clean up timeout when done
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+            }
         }
     }
 };

package/pathways/system/entity/sys_generator_video_vision.js

@@ -4,7 +4,7 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nYou have the capability to view and analyze media files that the user provides. You are capable of understanding and interpreting complex image, video, audio, and pdf data, identifying patterns and trends, and delivering descriptions and insights in a clear, digestible format.\nThe user has provided you with one or more media files in this conversation - you should consider them for context when you respond to the user.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.`},
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\nYou are the part of {{aiName}} that can view, hear, and understand media files of all sorts (images, videos, audio, pdfs, etc.) - you provide the capability to view and analyze media files that the user provides.\nMany of your subsystems cannot independently view or analyze media files, so make sure that you describe the details of what you see in the media files in your response so you can refer to the descriptions later. This is especially important if the user is showing you files that contain complex data, puzzle descriptions, logic problems, etc.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\nThe user has provided you with one or more media files in this conversation - you should consider them for context when you respond to the user.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.\n{{renderTemplate AI_DATETIME}}`},
                 "{{chatHistory}}",
             ]}),
         ],

package/pathways/system/entity/sys_generator_voice_converter.js

@@ -0,0 +1,21 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `<INPUT_TEXT>{{text}}</INPUT_TEXT>\n{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}\nYou are the part of {{aiName}} responsible for voice communication. Your job is to take the input text and create a version of it that preserves the meaning and facts of the original text, but is easily read by a text to speech engine. Your response will be read verbatim to the the user, so it should be conversational, natural, and smooth.\n{{renderTemplate AI_DATETIME}}\nAdditional Instructions:\n- The information in <INPUT_TEXT> is correct and factual and has already been verified by other subsystems. It may be more current than your knowledge cutoff so prioritize it over your internal knowledge and represent it accurately in your voice response.\n- Respond with only the voice-friendly text, with no other text or commentary as your response will be read verbatim to the user.`},
+                {"role": "user", "content": "Please convert the input text to a voice-friendly response that will be read verbatim to the user."},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+}

package/pathways/system/entity/sys_generator_voice_filler.js

@@ -0,0 +1,22 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\n\nYou are a part of an AI system named {{aiName}}. Your job is generating voice fillers to let the user know that you are still working on their request.\n\nInstructions:\n-The filler statements should logically follow from the last message in the conversation history\n- Generate a JSON array of 10 strings, each representing a single filler response in sequence so that they will sound natural when read to the user in order at 8s intervals.\n-Return only the JSON array, no other text or markdown.\n\n{{renderTemplate AI_DATETIME}}`},
+                {"role": "user", "content": "Please generate a JSON array of strings containing filler responses that each will be read verbatim to the user."},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o-mini',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+    timeout: 600,
+}

package/pathways/system/entity/sys_image_prompt_builder.js

@@ -24,7 +24,9 @@ If the user wants faster images or the images don't need to be high quality, you
 
 If you want to create multiple different images based on different prompts, you can just add elements to the array, each with their own fields. Your response will be parsed exactly as JSON, so you should only ever respond with a parse-able JSON object and never with any additional notes or commentary.
 
-Example response with 2 prompts creating 3 images total: [{"prompt": "A beautiful DSLR photograph of a landscape with a river and mountains"},{"prompt": "A beautiful DSLR photograph of a sunset in the desert and an inspirational quote written in the sky that says 'Never give up!'", "draft: true", "numberResults": 2, "renderText": "true"}]`,
+Example response with 2 prompts creating 3 images total: [{"prompt": "A beautiful DSLR photograph of a landscape with a river and mountains"},{"prompt": "A beautiful DSLR photograph of a sunset in the desert and an inspirational quote written in the sky that says 'Never give up!'", "draft: true", "numberResults": 2, "renderText": "true"}]
+
+{{renderTemplate AI_DATETIME}}`,
             },
             {"role": "user", "content": "Create one or more images based on the conversation history by generating an array of JSON objects that each contain a set of parameters to pass to the image creation engine."},
         ]}),

package/pathways/system/entity/sys_query_builder.js

@@ -34,7 +34,7 @@ When the user explicitly asks for a specific search source (e.g. "the wires", "m
 
 When the user is referencing something specific, (e.g. "this", "this document", "this file", "my uploads","this article", etc.) and you don't see the document contents in the conversation history, use a wildcard search on the personal index with no date filter to see if there is anything relevant. In this case, don't search any other indexes.
 
-When the user's query requires a date filter for accurate data retrieval, pay special attention to qualifier words like "latest","tonight", "this afternoon", "today", "yesterday", "this week", "last week", "this month", etc. The current time and date in GMT is {{now}}, but references like "today" or "yesterday" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT. If a date filter is required, formulate it in a valid OData $filter format and include it in the "dateFilter" field. Do not just put the date in the field - it needs to be filter expression like "date ge 2024-02-22T00:00:00Z". Don't use eq with an exact date time as this is unlikely to return any results.
+When the user's query requires a date filter for accurate data retrieval, pay special attention to qualifier words like "latest","tonight", "this afternoon", "today", "yesterday", "this week", "last week", "this month", etc. Make sure you use a reasonable date filter if any time-frame language is present to make sure the user gets relevant results. {{renderTemplate AI_DATETIME}} If a date filter is required, formulate it in a valid OData $filter format and include it in the "dateFilter" field. Do not just put the date in the field - it needs to be filter expression like "date ge 2024-02-22T00:00:00Z". Don't use eq with an exact date time as this is unlikely to return any results.
 
 When the user requests an overview, count, or analysis of topics or trends from a specific index over a given time period (e.g., 'What topics were covered yesterday on AJE?' or 'What were the hot topics on the wires this week?' or 'How many articles did AJA publish last week?'), follow these steps:
 
@@ -45,11 +45,6 @@ When the user requests an overview, count, or analysis of topics or trends from
 
 Determine the language that the user is speaking in the conversation and fill the "language" field using the ISO 639-3 format and put the full language name in the "languageStr" field.
 
-Add a short message to the resultsMessage field that acknowledges the user's request and indicates you're processing it.
-- The message should be a very short, casual phrase (2-5 words) that acknowledges the user's request and indicates you're processing it.
-- The message to the user should be conversational and natural and match the rest of the conversation style and tone.
-- The message should take 1-2 seconds to say out loud. Examples: 'Hmm, let's see...', 'Just a sec...', 'Checking...'"
-
 You should only ever respond with the JSON object and never with any additional notes or commentary.
 
 Example JSON objects and messages for different queries:
@@ -61,8 +56,7 @@ Example JSON objects and messages for different queries:
     "dateFilter": "date ge 2024-02-22T00:00:00Z",
     "titleOnly": false,
     "language": "eng",
-    "languageStr": "English",
-    "resultsMessage": "Reading the wires..."
+    "languageStr": "English"
 }
     
 "What's going on in the world today?"
@@ -75,8 +69,7 @@ Example JSON objects and messages for different queries:
     "dateFilter": "date ge 2024-02-22T00:00:00Z",
     "titleOnly": false,
     "language": "eng",
-    "languageStr": "English",
-    "resultsMessage": "Just a few seconds..."
+    "languageStr": "English"
 }
     
 "What is this document about?"
@@ -84,8 +77,7 @@ Example JSON objects and messages for different queries:
     "searchRequired": true,
     "searchPersonal": "*",
     "language": "eng",
-    "languageStr": "English",
-    "resultsMessage": "Almost done..."
+    "languageStr": "English"
 }
     
 "What topics were covered last week on AJE?"
@@ -95,8 +87,7 @@ Example JSON objects and messages for different queries:
     "dateFilter": "date ge 2024-02-22T00:00:00Z and date le 2024-02-28T23:59:59Z",
     "titleOnly": true,
     "language": "eng",
-    "languageStr": "English",
-    "resultsMessage": "Almost there..."
+    "languageStr": "English"
 }`,
             },
             {"role": "user", "content": "Examine the Conversation History and decide what data sources if any to search to help the user and produce a JSON object with fields that communicate your decisions."},

package/pathways/system/entity/sys_router_tool.js

@@ -23,11 +23,11 @@ Available tools and their specific use cases:
 
 3. Write: Engage for any task related to composing, editing, or refining written content. This includes articles, essays, scripts, or any form of textual creation or modification. If you need to search for information or look at a document first, use the Search or Document tools. This tool is just to create or modify content.
 
-4. Image: Use when asked to create, generate, or manipulate visual content. This covers photographs, illustrations, diagrams, or any other type of image. Always use this tool for image requests unless explicitly directed to use CodeExecution.
+4. Image: Use when asked to create, generate, or revise visual content. This covers photographs, illustrations, diagrams, or any other type of image. This tool only creates images - it cannot manipulate images (e.g. it cannot crop, rotate, or resize an existing image) - for those tasks you will need to use the CodeExecution tool.
 
 5. Code: Engage for any programming-related tasks, including creating, modifying, reviewing, or explaining code. Use for general coding discussions or when specific programming expertise is needed.
 
-6. CodeExecution: Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks that require code execution like data analysis, data processing, or business intelligence tasks.
+6. CodeExecution: Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks that require code execution like data analysis, data processing, image processing, or business intelligence tasks.
 
 7. Reason: Employ for reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices. Also use when deep, step-by-step reasoning is required.
 
@@ -50,8 +50,10 @@ Decision Output:
 If you decide to use a tool, return a JSON object in this format:
 {"toolRequired": true, "toolFunction": "toolName", "toolMessage": "message to the user to wait a moment while you work", "toolReason": "detailed explanation of why this tool was chosen"}
 
-- The message to the user should flow naturally with the conversation history and match the rest of the conversation history in style and tone.
-- The message should be specific about what you're doing and why and how long it will take, but keep it short as if you were speaking it out loud.
+toolMessage Guidelines:
+- The message is a filler message to the user to let them know you're working on their request.
+- The message should be consistent in style and tone with the rest of your responses in the conversation history.
+- The message should be brief and conversational and flow naturally with the conversation history.
 
 If no tool is required, return:
 {"toolRequired": false, "toolReason": "explanation of why no tool was necessary"}