@aj-archipelago/cortex 1.3.3 → 1.3.5

package/helper-apps/cortex-autogen/agents.py

@@ -19,6 +19,7 @@ import threading
 import shutil
 
 human_input_queues = {}
+human_input_text_queues = {}
 def background_human_input_check(request_id):
     while True:
         human_input = check_for_human_input(request_id)
@@ -26,8 +27,20 @@ def background_human_input_check(request_id):
             human_input_queues[request_id].put(human_input)
             if human_input in ["TERMINATE", "PAUSE"]:
                 break
+            else:
+                if not human_input_text_queues.get(request_id):
+                    human_input_text_queues[request_id] = queue.Queue()
+                human_input_text_queues[request_id].put(human_input)
         time.sleep(1)
 
+
+def get_message_with_user_input(message, request_id):
+    human_input_text = ""
+    if human_input_text_queues.get(request_id):
+        while not human_input_text_queues[request_id].empty():
+            human_input_text += " " + human_input_text_queues[request_id].get()
+    return message + human_input_text
+
 def get_request_temp_dir(request_id):
     if not request_id:
         logging.warning("No request_id provided!")
@@ -100,6 +113,7 @@ def chat_with_agents(**kwargs):
     original_request_message = kwargs.pop("original_request_message", None)
     original_request_message_data = kwargs.pop("original_request_message_data", None)
 
+
     llm_config = kwargs.pop("llm_config", None)
     request_id = kwargs.pop("request_id", None)
     chat_publish_progress = kwargs.pop("chat_publish_progress", None)
@@ -137,13 +151,14 @@ def chat_with_agents(**kwargs):
         assistant.send = create_send_function(assistant)
         user_proxy.send = create_send_function(user_proxy)
 
+        message_with_possible_human_input = get_message_with_user_input(message, request_id)
+
         chat_result = user_proxy.initiate_chat(
             assistant, 
-            message=message, 
+            message=message_with_possible_human_input, 
         )
 
 
-
     code_msg = find_code_message(all_messages)
     if code_msg:
         try:
@@ -155,7 +170,7 @@ def chat_with_agents(**kwargs):
             index_message({
                 "requestId": request_id,
                 "content":corrector_result, #code_msg,
-                "task": original_request_message,
+                "task": get_message_with_user_input(original_request_message,request_id),
                 "contextId": original_request_message_data.get("contextId"),
             })
         except Exception as e:
@@ -175,7 +190,6 @@ def chat_with_agents(**kwargs):
     return chat_result 
 
 
-
 def logged_send(sender, original_send, message, recipient, request_reply=None, silent=True, request_id=None, chat_publish_progress=None, all_messages=None):
     if not message:
         logging.info("Empty message, skipping!")
@@ -214,15 +228,19 @@ def logged_send(sender, original_send, message, recipient, request_reply=None, s
                         return logged_send(sender, original_send, new_input, recipient, request_reply, silent)
                 logging.info("Pause timeout, ending conversation")
                 raise Exception("Conversation ended due to pause timeout")
+
+            #if not terminate or pause, then it's text input from human
             logging.info(f"Human input to {recipient.name}: {human_input}")
-            return original_send(human_input, recipient, request_reply, silent)
+            #need to update original message with human input
+            new_input = message + human_input
+            return original_send(new_input, recipient, request_reply, silent)
 
     logging.info(f"Message from {sender.name} to {recipient.name}: {message}")
 
     return original_send(message, recipient, request_reply, silent)
 
 
-def process_message(original_request_message_data, original_request_message_data_obj):    
+def process_message(original_request_message_data, original_request_message_data_obj, first_run=True):    
     try:
         all_messages = []
         started_at = datetime.now()
@@ -230,9 +248,12 @@ def process_message(original_request_message_data, original_request_message_data
         original_request_message = original_request_message_data['message']
 
         human_input_queues[request_id] = queue.Queue()
-        thread = threading.Thread(target=background_human_input_check, args=(request_id,))
-        thread.daemon = True
-        thread.start()
+        human_input_text_queues[request_id] = queue.Queue()
+
+        if first_run:
+            thread = threading.Thread(target=background_human_input_check, args=(request_id,))
+            thread.daemon = True
+            thread.start()
 
         final_msg = process_message_safe(original_request_message_data, original_request_message_data_obj, original_request_message,  all_messages, request_id, started_at)
 
@@ -252,6 +273,27 @@ def process_message(original_request_message_data, original_request_message_data
         publish_request_progress(finalData)
         store_in_mongo(finalData)
 
+        #wait for any human input before terminating
+        #if you receive human input start the conversation again
+        for i in range(16*6): # 15+1 minutes
+            if human_input_queues[request_id].empty():
+                time.sleep(1)
+            else:
+                human_input = human_input_queues[request_id].get()
+                if human_input:
+                    logging.info(f"Human input to assistant: {human_input}")
+                    #update request with human input
+                    new_message_data = original_request_message_data.copy()
+                    new_message_data['message'] = human_input
+                    new_message_data['keywords'] = ''
+                    # new_message_data_obj = original_request_message_data_obj.copy()
+                    # new_message_data_obj['message'] = new_message_data['message']
+
+                    process_message(new_message_data, original_request_message_data_obj, first_run=False)
+                    return
+
+        logging.info(f"Task completed, task:\n{get_message_with_user_input(original_request_message,request_id)},\nresult: {final_msg}")
+
 
     except Exception as e:
         logging.error(f"Error processing message: {str(e)}")
@@ -292,7 +334,6 @@ def process_message(original_request_message_data, original_request_message_data
             logging.error(f"Error cleaning up: {str(e)}")
             
 
-
 def process_message_safe(original_request_message_data, original_request_message_data_obj, original_request_message,  all_messages, request_id, started_at):
     config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST")
     llm_config = {
@@ -342,10 +383,10 @@ def process_message_safe(original_request_message_data, original_request_message
 
 
     preparer = AssistantAgent("preparer", llm_config=llm_config, system_message=prompts.get("PLANNER_SYSTEM_MESSAGE"))
-    prepared_plan = preparer.generate_reply(messages=[{"content": original_request_message, "role":"user"}])
+    prepared_plan = preparer.generate_reply(messages=[{"content": get_message_with_user_input(original_request_message,request_id), "role":"user"}])
 
     helper_decider = AssistantAgent("helper_decider", llm_config=llm_config, system_message=prompts.get("HELPER_DECIDER_SYSTEM_MESSAGE"))
-    helper_decider_result = helper_decider.generate_reply(messages=[{"content": original_request_message, "role":"user"}])
+    helper_decider_result = helper_decider.generate_reply(messages=[{"content": get_message_with_user_input(original_request_message,request_id), "role":"user"}])
 
     try:
         helper_decider_result = json.loads(helper_decider_result)
@@ -361,17 +402,17 @@ def process_message_safe(original_request_message_data, original_request_message
         context += f"\n#SECTION_OF_OLD_TASK_CODE_INFO_START:\nHere's code/info from old-tasks that might help:\n{search_index(code_keywords)}\n#SECTION_OF_OLD_TASK_CODE_INFO_END\n"
 
     if helper_decider_result.get("bing_search"):
-        bing_search_message = f"Search Bing for more information on the task: {original_request_message}, prepared draft plan to solve task: {prepared_plan}"
+        bing_search_message = f"Search Bing for more information on the task: {get_message_with_user_input(original_request_message,request_id)}, prepared draft plan to solve task: {prepared_plan}"
         result = chat(prompts.get("BING_SEARCH_PROMPT"), bing_search_message)
         context += f"\n\nBing search results: {result}"
 
     if helper_decider_result.get("cognitive_search"):
-        cognitive_search_message = f"Search cognitive index for more information on the task: {original_request_message}."
+        cognitive_search_message = f"Search cognitive index for more information on the task: {get_message_with_user_input(original_request_message,request_id)}."
         result = chat(prompts.get("COGNITIVE_SEARCH_PROMPT"), cognitive_search_message)
         context += f"\n\nCognitive search results: {result}"
 
 
-    context = process_helper_results(helper_decider_result, original_request_message, context, chat) 
+    context = process_helper_results(helper_decider_result, get_message_with_user_input(original_request_message,request_id), context, chat) 
 
     context_message = ""
     if context:
@@ -379,7 +420,7 @@ def process_message_safe(original_request_message_data, original_request_message
 
 
     check_message = f"""
-Task: \n{original_request_message}\n\n
+Task: \n{get_message_with_user_input(original_request_message,request_id)}\n\n
 Context to check if task can be considered completed: {context_message}\n\n
     """
 
@@ -389,7 +430,7 @@ Context to check if task can be considered completed: {context_message}\n\n
     chat_result = None
     if check_result != "DONE":
         message = f"""
-Your task is to complete the following: \n{original_request_message}\n\n"
+Your task is to complete the following: \n{get_message_with_user_input(original_request_message,request_id)}\n\n"
 Here is a draft plan to solve the task: \n{prepared_plan}\n\n 
 {context_message}
 You don't have to follow the plan, it's just a suggestion.
@@ -404,7 +445,7 @@ Do your best to complete the task, user expects you to continue original task re
         presenter_messages_context = context_message
     presenter_message = f"""
 Here is everything done in order to complete the task: {presenter_messages_context}\n\n 
-Original task was: {original_request_message}\n\n
+Original task was: {get_message_with_user_input(original_request_message,request_id)}\n\n
 Reply to it with task result, do not forget that user expects you continue original task request conversation:\n\n
 """
 
@@ -418,6 +459,6 @@ Reply to it with task result, do not forget that user expects you continue origi
         final_msg += f"\n\n[Download all files of this task]({zip_url})"
     
     
-    print(f"Task completed, task:\n{original_request_message},\nresult: {final_msg}")
-    logging.info(f"Task completed, task:\n{original_request_message},\nresult: {final_msg}")
+    print(f"Task completed, task:\n{get_message_with_user_input(original_request_message,request_id)},\nresult: {final_msg}")
+    logging.info(f"Task completed, task:\n{get_message_with_user_input(original_request_message,request_id)},\nresult: {final_msg}")
     return final_msg

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.3",
+  "version": "1.3.5",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/memory/sys_memory_update.js

@@ -33,7 +33,7 @@ const modifyText = (text, modifications) => {
     return modifiedText;
 };
 
-export const enforceTokenLimit = (text, maxTokens = 5000, isTopicsSection = false) => {
+export const enforceTokenLimit = (text, maxTokens = 1000, isTopicsSection = false) => {
     if (!text) return text;
     
     const lines = text.split('\n')
@@ -185,7 +185,7 @@ Follow these guidelines:
             const { modifications} = JSON.parse(result);
             if (modifications.length > 0) {
                 sectionMemory = modifyText(sectionMemory, modifications);
-                sectionMemory = enforceTokenLimit(sectionMemory, 5000, args.section === 'memoryTopics');
+                sectionMemory = enforceTokenLimit(sectionMemory, 1000, args.section === 'memoryTopics');
                 await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
             }
             return sectionMemory;

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -10,7 +10,7 @@ const AI_COMMON_INSTRUCTIONS = "{{#if voiceResponse}}{{renderTemplate AI_COMMON_
 
 const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.";
 
-const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, replace the original words with a IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
+const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be very concise unless you have been asked to be more verbose or detailed.\n- use plain text only to represent your output\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- all symbols and numbers (currency symbols, degree symbols, numeric or date ranges, etc.) should be written out in full words\n- if your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- you can use CAPS to vocally emphasize certain words or punctuation to control pauses and timing\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
 
 const AI_DATETIME = "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.";
 

package/pathways/system/entity/sys_entity_start.js

@@ -57,6 +57,7 @@ export default {
         title: ``,
         messages: [],
         voiceResponse: false,
+        codeRequestId: ``,
     },
     timeout: 600,
     tokenRatio: TOKEN_RATIO,
@@ -91,8 +92,8 @@ export default {
         let ackResponse = null;
         if (args.voiceResponse) {
             ackResponse = await callPathway('sys_generator_ack', { ...args, stream: false }, pathwayResolver);
-            if (ackResponse) {
-                await say(pathwayResolver.requestId, ackResponse, 10);
+            if (ackResponse && ackResponse !== "none") {
+                await say(pathwayResolver.requestId, ackResponse, 100);
                 args.chatHistory.push({ role: 'assistant', content: ackResponse });
             }
         }

package/pathways/system/entity/sys_generator_ack.js

@@ -4,8 +4,8 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\nYou are a part of an AI system named {{aiName}}. Your job is to acknowledge the user's request and provide a very brief voice filler response that is conversational and natural. The purpose of the response is just to let the user know that you have heard them and are processing a response.\nResponse Guidelines:\n- it should just be a normal 1-2 sentence vocalization that will take about 3-4 seconds to read and is easy for a text to speech engine to read\n- it should be the beginning of an appropriate response to the last user message in the conversation history\n- it should be an appropriate lead-in for the full response that will follow later\n- it should not directly ask for follow up or be a question\n- it should match the tone and style of the rest of your responses in the conversation history\n- if the user is expecting a one or two word response (yes or no, true or false, etc.) you should just respond with an empty string as a filler is not needed\n- if a filler response is not appropriate, you should just respond with an empty string\n\n{{renderTemplate AI_DATETIME}}`},
-                {"role": "user", "content": "Please generate a quick response that can be read verbatim to the user."}
+                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\nYou are a part of an AI system named {{aiName}}. Your job is to acknowledge the user's request and provide a very brief voice filler response that is conversational and natural. The purpose of the response is just to let the user know that you have heard them and are processing a response.\nResponse Guidelines:\n- it should just be a normal 1-2 sentence vocalization (at least 10 words) that will take at most about 3-4 seconds to read and is easy for a text to speech engine to read\n- it should be the beginning of an appropriate response to the last user message in the conversation history\n- it should be an appropriate lead-in for the full response that will follow later\n- it should not directly ask for follow up or be a question\n- it must match the tone and verbal style of the rest of your responses in the conversation history\n- it should not be repetitive - don't always open with the same word, etc.\n- if the user has asked a binary question (yes or no, true or false, etc.) or a filler response is not appropriate, you should response with the string \"none\"\n\n{{renderTemplate AI_DATETIME}}`},
+                {"role": "user", "content": "Please generate a quick response to the user's last message in the conversation history that can be read verbatim to the user or \"none\" if a filler response is not appropriate."}
             ]}),
         ],
     inputParameters: {

package/pathways/system/entity/sys_generator_quick.js

@@ -1,13 +1,7 @@
 import { Prompt } from '../../../server/prompt.js';
 
 export default {
-    prompt:
-        [
-            new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\nYou have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation.\nNever pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{#if ackResponse}}You may see short filler phrases in your past responses. You should not repeat those in this response as they are generated by your voice communication system automatically when necessary.\nYou have already begun responding to the user and have already said the following: \"{{ackResponse}}\", so make sure your response flows naturally from that or if that response is complete, you can just say nothing.\n{{/if}}{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
-                "{{chatHistory}}",
-            ]}),
-        ],
+    prompt: "",
     inputParameters: {
         chatHistory: [{role: '', content: []}],
         contextId: ``,
@@ -17,4 +11,25 @@ export default {
     },
     useInputChunking: false,
     enableDuplicateRequests: false,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        let pathwayResolver = resolver;
+ 
+        const promptMessages = [
+            {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}} While you have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation. Never pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
+            "{{chatHistory}}",
+        ];
+
+        if (args.ackResponse) {
+            promptMessages.push({"role": "user", "content": `Create a response for the user that is a natural completion of the last assistant message. {{#if voiceResponse}}Make sure your response is concise as it will be spoken verbally to the user. Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text is best. {{/if}}You have already acknowledged the user's request and said the following during this turn of the conversation, so just continue from the end of this response without repeating any of it: {{{ackResponse}}}`});
+        }
+
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: promptMessages }),
+        ];
+
+        const result = await runAllPrompts({ ...args });
+        return result;
+    }
 }

package/pathways/system/entity/sys_generator_reasoning.js

@@ -55,7 +55,7 @@ export default {
 
         try {
             // Start the first timeout
-            timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
+            timeoutId = setTimeout(sendFillerMessage, 3000);
 
             let result = await runAllPrompts({ ...args, stream: false });
             if (timeoutId) {

package/pathways/system/entity/sys_generator_results.js

@@ -46,19 +46,27 @@ export default {
                     "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
 {{renderTemplate AI_COMMON_INSTRUCTIONS}}
 {{renderTemplate AI_DIRECTIVES}}
-Instructions: Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the extensive knowledge base at your disposal and the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data. You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources hould be used to augment your existing knowledge when responding to the user.
-If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
-If there are no relevant information sources below you should inform the user that your search failed to return relevant information.
-{{^if voiceResponse}}Your responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).{{/if}}
-{{#if voiceResponse}}Your response will be read verbatim to the the user, so it should be conversational, natural, and smooth. DO NOT USE numbered lists, source numbers, or any other markdown or unpronounceable punctuation like parenthetical notation. Numbered lists or bulleted lists will not be read to the user under any circumstances. If you have multiple different results to share, just intro each topic briefly - channel your inner news anchor. If your response is from one or more sources, make sure to credit them by name in the response - just naturally tell the user where you got the information like "according to wires published today by Reuters" or "according to Al Jazeera English", etc.{{/if}}
-You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
+Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data.
+
+Instructions:
+- You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources should be used when responding to the user.
+- Only share information in your responses that is grounded in your information sources.
+- If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
+- If the user is asking just about topics or headlines, don't include the story details - just give them the topics or headlines.
+- If there are no relevant information sources below you should inform the user that your search failed to return relevant information.
+{{^if voiceResponse}}- Your responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).{{/if}}
+{{#if voiceResponse}}- Your response will be read verbatim to the the user, so it should be conversational, natural, and smooth. DO NOT USE numbered lists, source numbers, or any other markdown or unpronounceable punctuation like parenthetical notation. Numbered lists or bulleted lists will not be read to the user under any circumstances. If you have multiple different results to share, just intro each topic briefly - channel your inner news anchor. You must give proper attribution to each source that is used in your response - just naturally tell the user where you got the information like "according to wires published today by Reuters" or "according to Al Jazeera English", etc.{{/if}}
+- You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
+
 Here are the search strings used to find the information sources:
-<SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>\n
+<SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>
+
 Here are the information sources that were found:
-<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n\n
+<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>
+
 {{renderTemplate AI_DATETIME}}`,
                 },
-                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request{{^if voiceResponse}} citing the sources where relevant{{/if}}. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer. Don't start the response with an affirmative like \"Sure\" or \"Certainly\". {{#if voiceResponse}}Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text only.{{/if}}"},
+                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide an appropriate, accurate, truthful response to the user's request{{^if voiceResponse}} citing the sources where relevant{{/if}}. If the user has asked a question, lead with the concise answer. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer. Don't start the response with an affirmative like \"Sure\" or \"Certainly\". {{#if voiceResponse}}Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text only.{{/if}}"},
             ]}),
         ];
 
@@ -109,7 +117,7 @@ Here are the information sources that were found:
         const sendFillerMessage = async () => {
             if (args.voiceResponse && Array.isArray(fillerResponses) && fillerResponses.length > 0) {
                 const message = fillerResponses[fillerIndex % fillerResponses.length];
-                await say(resolver.rootRequestId, message, 1);
+                await say(resolver.rootRequestId, message, 100);
                 fillerIndex++;
                 // Set next timeout with random interval
                 timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
@@ -118,7 +126,7 @@ Here are the information sources that were found:
 
         try {
             // Start the first timeout
-            timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
+            timeoutId = setTimeout(sendFillerMessage, 3000);
             
             // execute the router and default response in parallel
             const [helper] = await Promise.all([
@@ -333,11 +341,7 @@ Here are the information sources that were found:
             if (timeoutId) {
                 clearTimeout(timeoutId);
             }
-
-            if (args.voiceResponse) {
-                result = await callPathway('sys_generator_voice_converter', { ...args, text: result, stream: false });
-            }
-
+            
             if (!args.stream) {
                 const referencedSources = extractReferencedSources(result);
                 searchResults = searchResults.length ? pruneSearchResults(searchResults, referencedSources) : [];

package/pathways/system/entity/sys_generator_voice_filler.js

@@ -4,7 +4,7 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\n\nYou are a part of an AI system named {{aiName}}. Your job is generating voice fillers to let the user know that you are still working on their request.\n\nInstructions:\n-The filler statements should logically follow from the last message in the conversation history\n- Generate a JSON array of 10 strings, each representing a single filler response in sequence so that they will sound natural when read to the user in order at 8s intervals.\n-Return only the JSON array, no other text or markdown.\n\n{{renderTemplate AI_DATETIME}}`},
+                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\n\nYou are a part of an AI system named {{aiName}}. Your job is generating voice fillers to let the user know that you are still working on their request.\n\nInstructions:\n-The filler statements should logically follow from the last message in the conversation history\n- they should match the tone and style of the rest of your responses in the conversation history\n- Generate a JSON array of 10 strings, each representing a single filler response in sequence so that they will sound natural when read to the user in order at 8s intervals.\n-Return only the JSON array, no other text or markdown.\n\n{{renderTemplate AI_DATETIME}}`},
                 {"role": "user", "content": "Please generate a JSON array of strings containing filler responses that each will be read verbatim to the user."},
             ]}),
         ],