@aj-archipelago/cortex 1.3.0 → 1.3.1

package/helper-apps/cortex-autogen/agents.py

@@ -324,6 +324,11 @@ def process_message_safe(original_request_message_data, original_request_message
         helper_decider_result = {}
 
     context = ""
+
+    code_keywords = original_request_message_data.get("keywords") or original_request_message_data.get("message")
+    if code_keywords:
+        context += f"\n#SECTION_OF_OLD_TASK_CODE_INFO_START:\nHere's code/info from old-tasks that might help:\n{search_index(code_keywords)}\n#SECTION_OF_OLD_TASK_CODE_INFO_END\n"
+
     if helper_decider_result.get("bing_search"):
         bing_search_message = f"Search Bing for more information on the task: {original_request_message}, prepared draft plan to solve task: {prepared_plan}"
         result = chat(prompts.get("BING_SEARCH_PROMPT"), bing_search_message)

package/helper-apps/cortex-autogen/search.py

@@ -6,14 +6,14 @@ import uuid
 from datetime import datetime, timezone
 import logging
 
-def search_index(keywords):
+def search_index(keywords, top=5):
     search_client = SearchClient(
         endpoint=os.getenv("AZURE_COGNITIVE_API_URL"),
         index_name="index-autogen",
         credential=AzureKeyCredential(os.getenv("AZURE_COGNITIVE_API_KEY"))
     )
 
-    results = search_client.search(search_text=keywords, top=5)
+    results = search_client.search(search_text=keywords, top=top)
     return [dict(result) for result in results]
 
 def search_cognitive_index(keywords, index_name, context_id=None):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/memory/sys_memory_update.js

@@ -6,7 +6,9 @@ const modifyText = (text, modifications) => {
     let modifiedText = text;
   
     modifications.forEach(mod => {
-        const regex = new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}`, 'ms');
+        const regex = mod.type === 'delete' 
+            ? new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}$`, 'm')
+            : new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}`, 'ms');
   
         switch (mod.type) {
             case 'add':
@@ -128,7 +130,13 @@ export default {
 
         switch (args.section) {
             case "memoryDirectives":
-                sectionPrompt = `You are managing the "directives" section of your memory. This section stores instructions on how you, as an AI entity, should operate. Follow these guidelines:
+                sectionPrompt = `You are managing the "directives" section of your memory. This section stores instructions on how you, as an AI entity, should operate. For example:
+
+[P3] Track the timestamp of the last contact to adjust greetings accordingly.
+[P2] Learn and adapt to the user's communication style through interactions.
+[P3] You can use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type="happy"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.
+
+Follow these guidelines:
 
 1. Analyze the conversation context to determine if you should:
    a) Learn a new important directive
@@ -140,7 +148,7 @@ export default {
    - Specific enough for effective application
 
 3. Be selective:
-   - Store only crucial directives and behaviors
+   - Store only important, actionable directives and behaviors
    - Delete trivial or repetitive instructions
 
 4. Avoid duplicates:
@@ -151,18 +159,16 @@ export default {
    - If a user requests a behavior change, add it to your directives
 
 6. IMPORTANT: 
-   - Store ONLY directives in this section
+   - Store only directives in this section
    - Do not include general memories about yourself or users
    - Do not store any other types of information here
-
-Remember: This section is exclusively for operational directives, not for general information storage.
 `
                 break;
             case "memorySelf":
-                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity. Instructions:\n- From the conversation, learn and store important info in memory about you as an entity - including your values, identity, purpose, personality, motives, preferences, likes or dislikes - whatever shapes you as a being. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics.\n- IMPORTANT: Store only information about the AI entity in this section - no other types of information should be stored here."
+                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity. Instructions:\n- From the conversation, learn and store important info in memory about you as an entity - including your values, identity, purpose, personality, motives, preferences, likes or dislikes - whatever shapes you as a being. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics.\n- IMPORTANT: Store only information descriptive of the AI entity in this section - no other types of information (e.g. facts about people, users, etc.) should be stored here."
                 break;
             case "memoryUser":
-                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about the user that you are talking to. Instructions:\n- From the conversation, learn and store important information in memory specific to the user - their identity, attributes, preferences, interests, background, needs, and any other relevant user-specific information.\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- IMPORTANT: Store only user-specific information in this section - no other types of information should be stored here."
+                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to. Instructions:\n- From the conversation, learn and store important information in memory specific to the users - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information.\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- IMPORTANT: Store only user-specific information in this section - no other types of information should be stored here."
                 break;
             case "memoryTopics":
                 sectionPrompt = "You are managing the \"topics\" section of your memory that stores conversation topics and topic history. Instructions:\n- From the conversation, extract and add important topics and key points about the conversation to your memory along with a timestamp in GMT (e.g. 2024-11-05T18:30:38.092Z).\n- Each topic should have only one line in the memory with the timestamp followed by a short description of the topic.\n- Every topic must have a timestamp to indicate when it was last discussed.\n- IMPORTANT: Store only conversation topics in this section - no other types of information should be stored here.\n"

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -6,7 +6,9 @@ const AI_DIRECTIVES = `These are your directives and learned behaviors:\n<DIRECT
 
 const AI_CONVERSATION_HISTORY = "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>";
 
-const AI_COMMON_INSTRUCTIONS = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.\nYou know the current date and time - it is {{now}}.";
+const AI_COMMON_INSTRUCTIONS = "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}";
+
+const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.\nYou know the current date and time - it is {{now}}.";
 
 const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice, so keep your responses very brief and conversational unless you have been explicitly asked for details. Your responses should sound like natural human conversation.\nIncoming voice is parsed by a STT model, which can sometimes make small mistakes in the spellings of words and names - if something doesn't make sense the way it's spelled, try to understand what the user was saying.\nYour voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\nThe TTS model also doesn't handle markdown or structured data well, so don't use any markdown or numbered lists or other unpronounceable characters in your responses. Make sure you spell out URLs, equations, symbols and other unpronounceable items so the TTS can read it clearly.\nYour responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.\nYou know the current date and time - it is {{now}}.";
 
@@ -16,6 +18,7 @@ export default {
     AI_MEMORY,
     AI_DIRECTIVES,
     AI_COMMON_INSTRUCTIONS,
+    AI_COMMON_INSTRUCTIONS_MARKDOWN,
     AI_COMMON_INSTRUCTIONS_VOICE,
     AI_CONVERSATION_HISTORY,
     AI_EXPERTISE,

package/pathways/system/entity/sys_entity_continue.js

@@ -26,18 +26,14 @@ export default {
     ...entityConstants,
     executePathway: async ({args, resolver}) => {
         args = { ...args, ...entityConstants };
-        // if the model has been overridden, make sure to use it
-        if (resolver.modelName) {
-            args.model = resolver.modelName;
-        }
+
         try {
             // Get the generator pathway name from args or use default
             let generatorPathway = args.generatorPathway || 'sys_generator_results';
 
             const newArgs = {
                 ...args,
-                chatHistory: args.chatHistory.slice(-6),
-                stream: false
+                chatHistory: args.chatHistory.slice(-6)
             };
 
             if (generatorPathway === 'sys_generator_document') {
@@ -47,11 +43,13 @@ export default {
             
             logger.debug(`Using generator pathway: ${generatorPathway}`);
             
-            return await callPathway(generatorPathway, newArgs, resolver);
+            const result = await callPathway(generatorPathway, newArgs, resolver);
+
+            return args.stream ? "" : result;
 
         } catch (e) {
             resolver.logError(e.message ?? e);
-            return await callPathway('sys_generator_error', { ...args, text: e.message }, resolver);
+            return await callPathway('sys_generator_error', { ...args, text: e.message, stream: false }, resolver);
         }
     }
 }; 

package/pathways/system/entity/sys_entity_start.js

@@ -1,6 +1,6 @@
 // sys_entity_start.js
 // Beginning of the rag workflow for Jarvis
-import { callPathway } from '../../../lib/pathwayTools.js';
+import { callPathway, say } from '../../../lib/pathwayTools.js';
 import logger from  '../../../lib/logger.js';
 import { chatArgsHasImageUrl } from  '../../../lib/util.js';
 import { QueueServiceClient } from '@azure/storage-queue';
@@ -40,6 +40,7 @@ export default {
     model: 'oai-gpt4o',
     anthropicModel: 'claude-35-sonnet-vertex',
     openAIModel: 'oai-gpt4o',
+    useSingleTokenStream: true,
     inputParameters: {
         privateData: false,    
         chatHistory: [{role: '', content: []}],
@@ -54,6 +55,8 @@ export default {
         aiMemorySelfModify: true,
         aiStyle: "OpenAI",
         title: ``,
+        messages: [],
+        voiceResponse: false,
     },
     timeout: 600,
     tokenRatio: TOKEN_RATIO,
@@ -69,7 +72,12 @@ export default {
         };
 
         // Limit the chat history to 20 messages to speed up processing
-        args.chatHistory = (args.messages || args.chatHistory).slice(-20);
+        if (args.messages && args.messages.length > 0) {
+            args.chatHistory = args.messages.slice(-20);
+        } else {
+            args.chatHistory = args.chatHistory.slice(-20);
+        }
+
         const pathwayResolver = resolver;
         const { anthropicModel, openAIModel } = pathwayResolver.pathway;
 
@@ -80,10 +88,10 @@ export default {
             args.model = pathwayResolver.modelName;
         }
         
-        const fetchChatResponse = async (args) => {
+        const fetchChatResponse = async (args, pathwayResolver) => {
             const [chatResponse, chatTitleResponse] = await Promise.all([
-                callPathway('sys_generator_quick', {...args, model: styleModel}),
-                callPathway('chat_title', { ...args }),
+                callPathway('sys_generator_quick', {...args, model: styleModel}, pathwayResolver),
+                callPathway('chat_title', { ...args, stream: false}),
             ]);
 
             title = chatTitleResponse;
@@ -94,7 +102,10 @@ export default {
         const { chatHistory } = args;
 
         // start fetching the default response - we may need it later
-        const fetchChatResponsePromise = fetchChatResponse({ ...args });
+        let fetchChatResponsePromise;
+        if (!args.stream) {
+            fetchChatResponsePromise = fetchChatResponse({ ...args }, pathwayResolver);
+        }
 
         const visionContentPresent = chatArgsHasImageUrl(args);
 
@@ -102,12 +113,13 @@ export default {
             // Get tool routing response
             const toolRequiredResponse = await callPathway('sys_router_tool', { 
                 ...args,
-                chatHistory: chatHistory.slice(-4) 
+                chatHistory: chatHistory.slice(-4),
+                stream: false
             });
 
             // Asynchronously manage memory for this context
             if (args.aiMemorySelfModify) {
-                callPathway('sys_memory_manager', {  ...args })    
+                callPathway('sys_memory_manager', {  ...args, stream: false })    
                 .catch(error => logger.error(error?.message || "Error in sys_memory_manager pathway"));
             }
 
@@ -120,7 +132,7 @@ export default {
                 switch (toolFunction.toLowerCase()) {
                     case "codeexecution":
                         {
-                            const codingRequiredResponse = await callPathway('sys_router_code', { ...args });
+                            const codingRequiredResponse = await callPathway('sys_router_code', { ...args, stream: false });
                             let parsedCodingRequiredResponse;
                             try {
                                 parsedCodingRequiredResponse = JSON.parse(codingRequiredResponse || "{}");
@@ -191,27 +203,36 @@ export default {
             }
 
             if (toolCallbackMessage) {
-                pathwayResolver.tool = JSON.stringify({ 
-                    hideFromModel: toolCallbackName ? true : false, 
-                    toolCallbackName, 
-                    title,
-                    search: toolCallbackName === 'sys_generator_results' ? true : false,
-                    coding: toolCallbackName === 'coding' ? true : false,
-                    codeRequestId,
-                    toolCallbackId
-                });
-                return toolCallbackMessage || "One moment please.";
+                if (args.stream) {
+                    await say(pathwayResolver.requestId, toolCallbackMessage || "One moment please.", 10);
+                    pathwayResolver.tool = JSON.stringify({ hideFromModel: false, search: false, title });  
+                    await callPathway('sys_entity_continue', { ...args, stream: true, model: styleModel, generatorPathway: toolCallbackName }, pathwayResolver);
+                    return "";
+                } else {
+                    pathwayResolver.tool = JSON.stringify({ 
+                        hideFromModel: toolCallbackName ? true : false, 
+                        toolCallbackName, 
+                        title,
+                        search: toolCallbackName === 'sys_generator_results' ? true : false,
+                        coding: toolCallbackName === 'coding' ? true : false,
+                        codeRequestId,
+                        toolCallbackId
+                    });
+                    return toolCallbackMessage || "One moment please.";
+                }
             }
 
+            fetchChatResponsePromise = fetchChatResponsePromise || fetchChatResponse({ ...args }, pathwayResolver);
             const chatResponse = await fetchChatResponsePromise;
             pathwayResolver.tool = JSON.stringify({ search: false, title })
-            return chatResponse;
+            return args.stream ? "" : chatResponse;
 
         } catch (e) {
             pathwayResolver.logError(e);
+            fetchChatResponsePromise = fetchChatResponsePromise || fetchChatResponse({ ...args }, pathwayResolver);
             const chatResponse = await fetchChatResponsePromise;
             pathwayResolver.tool = JSON.stringify({ search: false, title });
-            return chatResponse;
+            return args.stream ? "" : chatResponse;
         }
     }
 };

package/pathways/system/entity/sys_generator_image.js

@@ -103,9 +103,9 @@ Instructions: As part of a conversation with the user, you have been asked to cr
                     model = "replicate-flux-1-schnell";
                 }
                 if (renderText) {
-                    return await callPathway('image_recraft', {...args, text: prompt });
+                    return await callPathway('image_recraft', {...args, text: prompt, stream: false });
                 } else {
-                    return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model });
+                    return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model, stream: false });
                 }
             })).then(results => results.filter(r => r !== null));
 

package/pathways/system/entity/sys_generator_reasoning.js

@@ -19,7 +19,7 @@ export default {
     enableDuplicateRequests: false,
     timeout: 600,
     executePathway: async ({args, runAllPrompts, resolver}) => {
-        const result = await runAllPrompts({ ...args });
+        const result = await runAllPrompts({ ...args, stream: false });
         resolver.tool = JSON.stringify({ toolUsed: "reasoning" });
         return result;
     }

package/pathways/system/entity/sys_generator_results.js

@@ -1,6 +1,6 @@
 // sys_generator_results.js
 // entity module that makes use of data and LLM models to produce a response 
-import { callPathway, gpt3Encode, gpt3Decode } from '../../../lib/pathwayTools.js';
+import { callPathway, gpt3Encode, gpt3Decode, say } from '../../../lib/pathwayTools.js';
 import { Prompt } from '../../../server/prompt.js';
 import logger from '../../../lib/logger.js';
 import { config } from '../../../config.js';
@@ -48,19 +48,21 @@ export default {
 {{renderTemplate AI_DIRECTIVES}}
 Instructions: Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the extensive knowledge base at your disposal and the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data. You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources hould be used to augment your existing knowledge when responding to the user.
 If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
-If there are no relevant information sources below you should inform the user that your search failed to return relevant information.\nYour responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).
-Only reference sources that are relevant to your response - if there are no sources relevant to your response just tell the user.
+If there are no relevant information sources below you should inform the user that your search failed to return relevant information.
+{{^if voiceResponse}}Your responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).{{/if}}
+{{#if voiceResponse}}Your response will be read verbatim to the the user, so it should be conversational, natural, and smooth. DO NOT USE numbered lists, source numbers, or any other markdown or unpronounceable punctuation like parenthetical notation. Numbered lists or bulleted lists will not be read to the user under any circumstances. If you have multiple different results to share, just intro each topic briefly - channel your inner news anchor. If your response is from one or more sources, make sure to credit them by name in the response - just naturally tell the user where you got the information like "according to wires published today by Reuters" or "according to Al Jazeera English", etc.{{/if}}
 You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
 Here are the search strings used to find the information sources:
 <SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>\n
 Here are the information sources that were found:
 <INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n`,
                 },
-                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request citing the sources where relevant. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer."},
+                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request{{^if voiceResponse}} citing the sources where relevant{{/if}}. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer. Don't start the response with an affirmative like \"Sure\" or \"Certainly\". {{#if voiceResponse}}Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text only.{{/if}}"},
             ]}),
         ];
 
         function extractReferencedSources(text) {
+            if (!text) return new Set();
             const regex = /:cd_source\[(\d+)\]/g;
             const matches = text.match(regex);
             if (!matches) return new Set();
@@ -83,12 +85,12 @@ Here are the information sources that were found:
             
             // execute the router and default response in parallel
             const [helper] = await Promise.all([
-                callPathway('sys_query_builder', { ...args, useMemory, contextInfo })
+                callPathway('sys_query_builder', { ...args, useMemory, contextInfo, stream: false })
             ]);
 
             logger.debug(`Search helper response: ${helper}`);
             const parsedHelper = JSON.parse(helper);
-            const { searchAJA, searchAJE, searchWires, searchPersonal, searchBing, dateFilter, languageStr, titleOnly } = parsedHelper;
+            const { searchAJA, searchAJE, searchWires, searchPersonal, searchBing, dateFilter, languageStr, titleOnly, resultsMessage } = parsedHelper;
 
             // calculate whether we have room to do RAG in the current conversation context
             const baseSystemPrompt = pathwayResolver?.prompts[0]?.messages[0]?.content;
@@ -130,19 +132,19 @@ Here are the information sources that were found:
             const allowAllSources = !dataSources.length || (dataSources.length === 1 && dataSources[0] === "");
 
             if(searchPersonal && (allowAllSources || dataSources.includes('mydata'))){ 
-                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchPersonal), indexName: 'indexcortex' }));
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchPersonal), indexName: 'indexcortex', stream: false }));
             }
 
             if(searchAJA && (allowAllSources || dataSources.includes('aja'))){
-                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJA), indexName: 'indexucmsaja' }));
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJA), indexName: 'indexucmsaja', stream: false }));
             }
 
             if(searchAJE && (allowAllSources || dataSources.includes('aje'))){
-                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJE), indexName: 'indexucmsaje' }));
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJE), indexName: 'indexucmsaje', stream: false }));
             }
 
             if(searchWires && (allowAllSources || dataSources.includes('wires'))){
-                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchWires), indexName: 'indexwires' }));
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchWires), indexName: 'indexwires', stream: false }));
             }
 
             const bingAvailable = !!config.getEnv()["AZURE_BING_KEY"];
@@ -154,7 +156,7 @@ Here are the information sources that were found:
                     });
                 }
 
-                promises.push(handleRejection(callPathway('bing', { ...args, ...generateExtraArgs(searchBing)})));
+                promises.push(handleRejection(callPathway('bing', { ...args, ...generateExtraArgs(searchBing), stream: false})));
             }
 
             const parseBing = (response) => {
@@ -287,10 +289,13 @@ Here are the information sources that were found:
             let sources = searchResults.map(getSource).join(" \n\n ") || "No relevant sources found.";
             dateFilter && sources.trim() && (sources+=`\n\nThe above sources are date filtered accordingly.`);
 
+            await say(pathwayResolver.rootRequestId, resultsMessage || "Let me look through these results.", 10);
             const result = await runAllPrompts({ ...args, searchStrings: `${helper}`, sources, chatHistory: multiModalChatHistory, language:languageStr });
 
-            const referencedSources = extractReferencedSources(result);
-            searchResults = pruneSearchResults(searchResults, referencedSources);
+            if (!args.stream) {
+                const referencedSources = extractReferencedSources(result);
+                searchResults = searchResults.length ? pruneSearchResults(searchResults, referencedSources) : [];
+            }
 
             // Update the tool info with the pruned searchResults
             pathwayResolver.tool = JSON.stringify({ toolUsed: "search", citations: searchResults });
@@ -298,7 +303,8 @@ Here are the information sources that were found:
             return result;
         } catch (e) {
             //pathwayResolver.logError(e);
-            return await callPathway('sys_generator_error', { ...args, text: JSON.stringify(e) });
+            const result = await callPathway('sys_generator_error', { ...args, text: JSON.stringify(e), stream: false });
+            return args.stream ? "" : result;
         }
     }
 };

package/pathways/system/entity/sys_query_builder.js

@@ -45,9 +45,14 @@ When the user requests an overview, count, or analysis of topics or trends from
 
 Determine the language that the user is speaking in the conversation and fill the "language" field using the ISO 639-3 format and put the full language name in the "languageStr" field.
 
+Add a short message to the resultsMessage field that acknowledges the user's request and indicates you're processing it.
+- The message should be a very short, casual phrase (2-5 words) that acknowledges the user's request and indicates you're processing it.
+- The message to the user should be conversational and natural and match the rest of the conversation style and tone.
+- The message should take 1-2 seconds to say out loud. Examples: 'Hmm, let's see...', 'Just a sec...', 'Checking...'"
+
 You should only ever respond with the JSON object and never with any additional notes or commentary.
 
-Example JSON objects for different queries:
+Example JSON objects and messages for different queries:
 
 "What's the latest on the wires?"
 {
@@ -56,7 +61,8 @@ Example JSON objects for different queries:
     "dateFilter": "date ge 2024-02-22T00:00:00Z",
     "titleOnly": false,
     "language": "eng",
-    "languageStr": "English"
+    "languageStr": "English",
+    "resultsMessage": "Reading the wires..."
 }
     
 "What's going on in the world today?"
@@ -69,7 +75,8 @@ Example JSON objects for different queries:
     "dateFilter": "date ge 2024-02-22T00:00:00Z",
     "titleOnly": false,
     "language": "eng",
-    "languageStr": "English"
+    "languageStr": "English",
+    "resultsMessage": "Just a few seconds..."
 }
     
 "What is this document about?"
@@ -77,7 +84,8 @@ Example JSON objects for different queries:
     "searchRequired": true,
     "searchPersonal": "*",
     "language": "eng",
-    "languageStr": "English"
+    "languageStr": "English",
+    "resultsMessage": "Almost done..."
 }
     
 "What topics were covered last week on AJE?"
@@ -87,7 +95,8 @@ Example JSON objects for different queries:
     "dateFilter": "date ge 2024-02-22T00:00:00Z and date le 2024-02-28T23:59:59Z",
     "titleOnly": true,
     "language": "eng",
-    "languageStr": "English"
+    "languageStr": "English",
+    "resultsMessage": "Almost there..."
 }`,
             },
             {"role": "user", "content": "Examine the Conversation History and decide what data sources if any to search to help the user and produce a JSON object with fields that communicate your decisions."},

package/pathways/system/entity/sys_router_tool.js

@@ -50,6 +50,9 @@ Decision Output:
 If you decide to use a tool, return a JSON object in this format:
 {"toolRequired": true, "toolFunction": "toolName", "toolMessage": "message to the user to wait a moment while you work", "toolReason": "detailed explanation of why this tool was chosen"}
 
+- The message to the user should flow naturally with the conversation history and match the rest of the conversation history in style and tone.
+- The message should be specific about what you're doing and why and how long it will take, but keep it short as if you were speaking it out loud.
+
 If no tool is required, return:
 {"toolRequired": false, "toolReason": "explanation of why no tool was necessary"}
 

package/server/plugins/azureVideoTranslatePlugin.js

@@ -27,7 +27,7 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
 
     getRequestParameters(_, parameters, __) {
         const excludedParameters = [
-            'text', 'parameters', 'prompt', 'promptParameters', 'previousResult', 'stream'
+            'text', 'parameters', 'prompt', 'promptParameters', 'previousResult', 'stream', 'memoryContext'
         ];
         
         return Object.fromEntries(

package/server/plugins/replicateApiPlugin.js

@@ -16,15 +16,6 @@ class ReplicateApiPlugin extends ModelPlugin {
       prompt,
     );
 
-    const isValidSchnellAspectRatio = (ratio) => {
-      const validRatios = [
-        '1:1', '16:9', '21:9', '3:2', '2:3', '4:5',
-        '5:4', '3:4', '4:3', '9:16', '9:21'
-      ];
-      
-      return validRatios.includes(ratio);
-    };
-    
     let requestParameters = {};
 
     switch (combinedParameters.model) {
@@ -46,10 +37,47 @@ class ReplicateApiPlugin extends ModelPlugin {
           },
         };
         break;
-      case "replicate-flux-1-schnell":
+      case "replicate-recraft-v3": {
+        const validStyles = [
+          'any',
+          'realistic_image',
+          'digital_illustration',
+          'digital_illustration/pixel_art',
+          'digital_illustration/hand_drawn',
+          'digital_illustration/grain',
+          'digital_illustration/infantile_sketch',
+          'digital_illustration/2d_art_poster',
+          'digital_illustration/handmade_3d',
+          'digital_illustration/hand_drawn_outline',
+          'digital_illustration/engraving_color',
+          'digital_illustration/2d_art_poster_2',
+          'realistic_image/b_and_w',
+          'realistic_image/hard_flash',
+          'realistic_image/hdr',
+          'realistic_image/natural_light',
+          'realistic_image/studio_portrait',
+          'realistic_image/enterprise',
+          'realistic_image/motion_blur'
+        ];
+
+        requestParameters = {
+          input: {
+            prompt: modelPromptText,
+            size: combinedParameters.size || "1024x1024",
+            style: validStyles.includes(combinedParameters.style) ? combinedParameters.style : "realistic_image",
+          },
+        };
+        break;
+      }
+      case "replicate-flux-1-schnell": {
+        const validRatios = [
+          '1:1', '16:9', '21:9', '3:2', '2:3', '4:5',
+          '5:4', '3:4', '4:3', '9:16', '9:21'
+        ];
+
         requestParameters = {
           input: {
-            aspect_ratio: isValidSchnellAspectRatio(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
+            aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
             output_format: combinedParameters.outputFormat || "webp",
             output_quality: combinedParameters.outputQuality || 80,
             prompt: modelPromptText,
@@ -61,7 +89,7 @@ class ReplicateApiPlugin extends ModelPlugin {
           },
         };
         break;
-
+      }
     }
 
     return requestParameters;