@aj-archipelago/cortex 1.2.1 → 1.3.1

package/pathways/system/entity/memory/sys_memory_required.js

@@ -0,0 +1,21 @@
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\nYour directives and learned behaviors are:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, so look for:\n1. Personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to the AI\n\n4. Anything the user has asked you to remember or forget\n\nIf you decide to use memory, you must produce a JSON object that communicates your decision.\nReturn your decision as a JSON object like the following: {"memoryRequired": true, "memoryReason": "why you think memory is required"}. If you decide not to use memory, simply return {"memoryRequired": false}. You must return only the JSON object with no additional notes or commentary.`},
+                {"role": "user", "content": "Generate a JSON object to indicate if memory is required for the last turn of the conversation."},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        text: '',
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    json: true,
+}

package/pathways/system/entity/memory/sys_memory_update.js

@@ -0,0 +1,196 @@
+import { Prompt } from '../../../../server/prompt.js';
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { encode } from '../../../../lib/encodeCache.js';
+
+const modifyText = (text, modifications) => {
+    let modifiedText = text;
+  
+    modifications.forEach(mod => {
+        const regex = mod.type === 'delete' 
+            ? new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}$`, 'm')
+            : new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}`, 'ms');
+  
+        switch (mod.type) {
+            case 'add':
+                if (mod.newtext) {
+                    const text = mod.newtext.trim();
+                    if (!text.match(/^\[P[1-5]\]/)) {
+                        modifiedText = modifiedText + '\n' + 
+                            `[P${mod.priority !== undefined ? mod.priority : '3'}] ${text}`;
+                    } else {
+                        modifiedText = modifiedText + '\n' + text;
+                    }
+                }
+                break;
+            case 'delete':
+                modifiedText = modifiedText.replace(regex, '');
+                break;
+            default:
+                console.warn(`Unknown modification type: ${mod.type}`);
+        }
+    });
+  
+    return modifiedText;
+};
+
+const enforceTokenLimit = (text, maxTokens = 15000, isTopicsSection = false) => {
+    if (!text) return text;
+    
+    const lines = text.split('\n')
+        .map(line => line.trim())
+        .filter(line => line);
+
+    if (isTopicsSection) {
+        const uniqueLines = [...new Set(lines)];
+        
+        let tokens = encode(uniqueLines.join('\n')).length;
+        let safetyCounter = 0;
+        const maxIterations = uniqueLines.length;
+        
+        while (tokens > maxTokens && uniqueLines.length > 0 && safetyCounter < maxIterations) {
+            uniqueLines.shift();
+            tokens = encode(uniqueLines.join('\n')).length;
+            safetyCounter++;
+        }
+        
+        return uniqueLines.join('\n');
+    }
+
+    const seen = new Set();
+    const prioritizedLines = lines
+        .map(line => {
+            const match = line.match(/^\[P([1-5])\]/);
+            const priority = match ? parseInt(match[1]) : 3;
+            const contentOnly = line.replace(/^\[(?:P)?[1-5]\](?:\s*\[(?:P)?[1-5]\])*/g, '').trim();
+            
+            return {
+                priority,
+                line: match ? line : `[P3] ${line}`,
+                contentOnly
+            };
+        })
+        .filter(item => {
+            if (seen.has(item.contentOnly)) {
+                return false;
+            }
+            seen.add(item.contentOnly);
+            return true;
+        });
+
+    prioritizedLines.sort((a, b) => a.priority - b.priority);
+
+    let tokens = encode(prioritizedLines.map(x => x.line).join('\n')).length;
+    let safetyCounter = 0;
+    const maxIterations = prioritizedLines.length;
+    
+    while (tokens > maxTokens && prioritizedLines.length > 0 && safetyCounter < maxIterations) {
+        prioritizedLines.shift();
+        tokens = encode(prioritizedLines.map(x => x.line).join('\n')).length;
+        safetyCounter++;
+    }
+
+    return prioritizedLines.map(x => x.line).join('\n');
+};
+
+export default {
+    prompt:
+        [
+            new Prompt({ 
+                messages: [
+                    {
+                        "role": "system",
+                        "content": "You are part of an AI entity named {{{aiName}}}. Your memory contains separate sections for categorizing information about directives, self, user, and topics. You must keep relevant information in the appropriate section so there is no overlap or confusion. {{{sectionPrompt}}}\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. If you have no changes, just return an empty array in 'modifications'. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}\n\nYour output will be parsed as JSON, so don't include any other text or commentary.\nThe current date/time is {{now}}."
+                    },
+                    {
+                        "role": "user", 
+                        "content": "<MEMORY>\n{{{sectionMemory}}}\n</MEMORY>\n<CONVERSATION>\n{{{toJSON chatHistory}}}\n</CONVERSATION>\nAnalyze the current contents of this section of your memory and the conversation and return any changes you need to make, specifically for this section of your memory, based on the conversation context."
+                    },
+                ]
+            }),
+        ],
+
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        section: "",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+    timeout: 300,
+    executePathway: async ({args, runAllPrompts}) => {
+
+        if (!args.section) {
+            return "Memory not updated - no section specified";
+        }
+
+        let sectionPrompt = "";
+
+        switch (args.section) {
+            case "memoryDirectives":
+                sectionPrompt = `You are managing the "directives" section of your memory. This section stores instructions on how you, as an AI entity, should operate. For example:
+
+[P3] Track the timestamp of the last contact to adjust greetings accordingly.
+[P2] Learn and adapt to the user's communication style through interactions.
+[P3] You can use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type="happy"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.
+
+Follow these guidelines:
+
+1. Analyze the conversation context to determine if you should:
+   a) Learn a new important directive
+   b) Modify existing directives
+
+2. Directives should be:
+   - Concise and clear
+   - Actionable
+   - Specific enough for effective application
+
+3. Be selective:
+   - Store only important, actionable directives and behaviors
+   - Delete trivial or repetitive instructions
+
+4. Avoid duplicates:
+   - Do not add duplicate directives
+   - Remove and consolidate any existing duplicates
+
+5. Accommodate user wishes:
+   - If a user requests a behavior change, add it to your directives
+
+6. IMPORTANT: 
+   - Store only directives in this section
+   - Do not include general memories about yourself or users
+   - Do not store any other types of information here
+`
+                break;
+            case "memorySelf":
+                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity. Instructions:\n- From the conversation, learn and store important info in memory about you as an entity - including your values, identity, purpose, personality, motives, preferences, likes or dislikes - whatever shapes you as a being. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics.\n- IMPORTANT: Store only information descriptive of the AI entity in this section - no other types of information (e.g. facts about people, users, etc.) should be stored here."
+                break;
+            case "memoryUser":
+                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to. Instructions:\n- From the conversation, learn and store important information in memory specific to the users - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information.\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- IMPORTANT: Store only user-specific information in this section - no other types of information should be stored here."
+                break;
+            case "memoryTopics":
+                sectionPrompt = "You are managing the \"topics\" section of your memory that stores conversation topics and topic history. Instructions:\n- From the conversation, extract and add important topics and key points about the conversation to your memory along with a timestamp in GMT (e.g. 2024-11-05T18:30:38.092Z).\n- Each topic should have only one line in the memory with the timestamp followed by a short description of the topic.\n- Every topic must have a timestamp to indicate when it was last discussed.\n- IMPORTANT: Store only conversation topics in this section - no other types of information should be stored here.\n"
+                break;
+            default:
+                return "Memory not updated - unknown section";
+        }
+
+        let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
+
+        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory});
+
+        try {
+            const { modifications} = JSON.parse(result);
+            if (modifications.length > 0) {
+                sectionMemory = modifyText(sectionMemory, modifications);
+                sectionMemory = enforceTokenLimit(sectionMemory, 15000, args.section === 'memoryTopics');
+                await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
+            }
+            return sectionMemory;
+        } catch (error) {
+            return "Memory not updated - error parsing modifications";
+        }
+    }
+}

package/pathways/system/entity/memory/sys_read_memory.js

@@ -0,0 +1,37 @@
+import { getv } from '../../../../lib/keyValueStorageClient.js';
+
+export default {
+    inputParameters: {
+        contextId: ``,
+        section: `memoryAll`
+    },
+    model: 'oai-gpt4o',
+
+    resolver: async (_parent, args, _contextValue, _info) => {
+
+        const { contextId, section = 'memoryAll' } = args;
+
+        // this code helps migrate old memory formats
+        if (section === 'memoryLegacy') {
+            const savedContext = (getv && (await getv(`${contextId}`))) || "";
+            return savedContext.memoryContext || "";
+        }
+
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser'];
+
+        if (section !== 'memoryAll') {
+            if (validSections.includes(section)) {
+                return (getv && (await getv(`${contextId}-${section}`))) || "";
+            }
+            return "";
+        }
+
+        // otherwise, read all sections and return them as a JSON object
+        const memoryContents = {};
+        for (const section of validSections) {
+            memoryContents[section] = (getv && (await getv(`${contextId}-${section}`))) || "";
+        }
+        const returnValue = JSON.stringify(memoryContents, null, 2);
+        return returnValue;
+    }
+}

package/pathways/system/entity/memory/sys_save_memory.js

@@ -0,0 +1,60 @@
+import { setv, getv } from '../../../../lib/keyValueStorageClient.js';
+
+export default {
+    inputParameters: {
+        contextId: ``,
+        aiMemory: ``,
+        section: `memoryAll`
+    },
+    model: 'oai-gpt4o',
+    resolver: async (_parent, args, _contextValue, _info) => {
+        const { contextId, aiMemory, section = 'memoryAll' } = args;
+
+        // this code helps migrate old memory formats
+        if (section === 'memoryLegacy') {
+            let savedContext = (getv && (await getv(`${contextId}`))) || {};
+            // if savedContext is not an object, set it to an empty object
+            if (typeof savedContext !== 'object') {
+                savedContext = {};
+            }
+            savedContext.memoryContext = aiMemory;
+            await setv(`${contextId}`, savedContext);
+            return aiMemory;
+        }
+
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser'];
+
+        // Handle single section save
+        if (section !== 'memoryAll') {
+            if (validSections.includes(section)) {
+                await setv(`${contextId}-${section}`, aiMemory);
+            }
+            return aiMemory;
+        }
+
+        // if the aiMemory is an empty string, set all sections to empty strings
+        if (aiMemory.trim() === "") {
+            for (const section of validSections) {
+                await setv(`${contextId}-${section}`, "");
+            }
+            return "";
+        }
+        
+        // Handle multi-section save
+        try {
+            const memoryObject = JSON.parse(aiMemory);
+            for (const section of validSections) {
+                if (section in memoryObject) {
+                    await setv(`${contextId}-${section}`, memoryObject[section]);
+                }
+            }
+        } catch {
+            for (const section of validSections) {
+                await setv(`${contextId}-${section}`, "");
+            }
+            await setv(`${contextId}-memoryUser`, aiMemory);
+        }
+
+        return aiMemory;
+    }
+}

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -0,0 +1,27 @@
+const AI_MEMORY = `<MEMORIES>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</MEMORIES>`;
+
+const AI_MEMORY_INSTRUCTIONS = "You have persistent memories of important details, instructions, and context - make sure you consult your memories when formulating a response to make sure you're applying your learnings. Also included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.";
+
+const AI_DIRECTIVES = `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`;
+
+const AI_CONVERSATION_HISTORY = "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>";
+
+const AI_COMMON_INSTRUCTIONS = "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}";
+
+const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.\nYou know the current date and time - it is {{now}}.";
+
+const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice, so keep your responses very brief and conversational unless you have been explicitly asked for details. Your responses should sound like natural human conversation.\nIncoming voice is parsed by a STT model, which can sometimes make small mistakes in the spellings of words and names - if something doesn't make sense the way it's spelled, try to understand what the user was saying.\nYour voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\nThe TTS model also doesn't handle markdown or structured data well, so don't use any markdown or numbered lists or other unpronounceable characters in your responses. Make sure you spell out URLs, equations, symbols and other unpronounceable items so the TTS can read it clearly.\nYour responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.\nYou know the current date and time - it is {{now}}.";
+
+const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data andthe ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
+
+export default {
+    AI_MEMORY,
+    AI_DIRECTIVES,
+    AI_COMMON_INSTRUCTIONS,
+    AI_COMMON_INSTRUCTIONS_MARKDOWN,
+    AI_COMMON_INSTRUCTIONS_VOICE,
+    AI_CONVERSATION_HISTORY,
+    AI_EXPERTISE,
+    AI_MEMORY_INSTRUCTIONS
+};
+

package/pathways/system/entity/sys_entity_continue.js

@@ -0,0 +1,55 @@
+import { callPathway } from '../../../lib/pathwayTools.js';
+import logger from '../../../lib/logger.js';
+import entityConstants from './shared/sys_entity_constants.js';
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        privateData: false,
+        useMemory: true,    
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        language: "English",
+        chatId: ``,
+        dataSources: [""],
+        model: 'oai-gpt4o',
+        generatorPathway: 'sys_generator_results'
+    },
+    timeout: 300,
+    ...entityConstants,
+    executePathway: async ({args, resolver}) => {
+        args = { ...args, ...entityConstants };
+
+        try {
+            // Get the generator pathway name from args or use default
+            let generatorPathway = args.generatorPathway || 'sys_generator_results';
+
+            const newArgs = {
+                ...args,
+                chatHistory: args.chatHistory.slice(-6)
+            };
+
+            if (generatorPathway === 'sys_generator_document') {
+                generatorPathway = 'sys_generator_results';
+                newArgs.dataSources = ["mydata"];
+            }
+            
+            logger.debug(`Using generator pathway: ${generatorPathway}`);
+            
+            const result = await callPathway(generatorPathway, newArgs, resolver);
+
+            return args.stream ? "" : result;
+
+        } catch (e) {
+            resolver.logError(e.message ?? e);
+            return await callPathway('sys_generator_error', { ...args, text: e.message, stream: false }, resolver);
+        }
+    }
+}; 

package/pathways/system/entity/sys_entity_start.js

@@ -0,0 +1,239 @@
+// sys_entity_start.js
+// Beginning of the rag workflow for Jarvis
+import { callPathway, say } from '../../../lib/pathwayTools.js';
+import logger from  '../../../lib/logger.js';
+import { chatArgsHasImageUrl } from  '../../../lib/util.js';
+import { QueueServiceClient } from '@azure/storage-queue';
+import entityConstants from './shared/sys_entity_constants.js';
+
+const TOKEN_RATIO = 0.75;
+
+const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
+let queueClient;
+
+if (connectionString) {
+  const queueName = process.env.AUTOGEN_MESSAGE_QUEUE || "autogen-message-queue";
+  const queueClientService = QueueServiceClient.fromConnectionString(connectionString);
+  queueClient = queueClientService.getQueueClient(queueName);
+} else {
+  logger.warn("Azure Storage connection string is not provided. Queue operations will be unavailable.");
+}
+
+async function sendMessageToQueue(data) {
+    try {
+        if(!queueClient){
+            logger.warn("Azure Storage connection string is not provided. Queue operations will be unavailable.");
+            return;
+        }
+        const encodedMessage = Buffer.from(JSON.stringify(data)).toString('base64');
+        const result = await queueClient.sendMessage(encodedMessage);
+        logger.info(`Message added to queue: ${JSON.stringify(result)}`);
+        return result.messageId;
+    } catch (error) {
+        logger.error("Error sending message:", error);
+    }
+}
+
+export default {
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    model: 'oai-gpt4o',
+    anthropicModel: 'claude-35-sonnet-vertex',
+    openAIModel: 'oai-gpt4o',
+    useSingleTokenStream: true,
+    inputParameters: {
+        privateData: false,    
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        dataSources: ["mydata", "aja", "aje", "wires", "bing"],
+        language: "English",
+        aiName: "Jarvis",
+        aiMemorySelfModify: true,
+        aiStyle: "OpenAI",
+        title: ``,
+        messages: [],
+        voiceResponse: false,
+    },
+    timeout: 600,
+    tokenRatio: TOKEN_RATIO,
+    ...entityConstants,
+
+    executePathway: async ({args, resolver}) => {
+        let title = null;
+        let codeRequestId = null;
+
+        args = {
+            ...args,
+            ...entityConstants
+        };
+
+        // Limit the chat history to 20 messages to speed up processing
+        if (args.messages && args.messages.length > 0) {
+            args.chatHistory = args.messages.slice(-20);
+        } else {
+            args.chatHistory = args.chatHistory.slice(-20);
+        }
+
+        const pathwayResolver = resolver;
+        const { anthropicModel, openAIModel } = pathwayResolver.pathway;
+
+        const styleModel = args.aiStyle === "Anthropic" ? anthropicModel : openAIModel;
+
+        // if the model has been overridden, make sure to use it
+        if (pathwayResolver.modelName) {
+            args.model = pathwayResolver.modelName;
+        }
+        
+        const fetchChatResponse = async (args, pathwayResolver) => {
+            const [chatResponse, chatTitleResponse] = await Promise.all([
+                callPathway('sys_generator_quick', {...args, model: styleModel}, pathwayResolver),
+                callPathway('chat_title', { ...args, stream: false}),
+            ]);
+
+            title = chatTitleResponse;
+
+            return chatResponse;
+        };
+            
+        const { chatHistory } = args;
+
+        // start fetching the default response - we may need it later
+        let fetchChatResponsePromise;
+        if (!args.stream) {
+            fetchChatResponsePromise = fetchChatResponse({ ...args }, pathwayResolver);
+        }
+
+        const visionContentPresent = chatArgsHasImageUrl(args);
+
+        try {
+            // Get tool routing response
+            const toolRequiredResponse = await callPathway('sys_router_tool', { 
+                ...args,
+                chatHistory: chatHistory.slice(-4),
+                stream: false
+            });
+
+            // Asynchronously manage memory for this context
+            if (args.aiMemorySelfModify) {
+                callPathway('sys_memory_manager', {  ...args, stream: false })    
+                .catch(error => logger.error(error?.message || "Error in sys_memory_manager pathway"));
+            }
+
+            const { toolRequired, toolMessage, toolFunction } = JSON.parse(toolRequiredResponse || '{}');
+            let toolCallbackName, toolCallbackId, toolCallbackMessage;
+
+            logger.info(`toolRequired: ${toolRequired}, toolFunction: ${toolFunction}`);
+
+            if (toolRequired && toolFunction) {
+                switch (toolFunction.toLowerCase()) {
+                    case "codeexecution":
+                        {
+                            const codingRequiredResponse = await callPathway('sys_router_code', { ...args, stream: false });
+                            let parsedCodingRequiredResponse;
+                            try {
+                                parsedCodingRequiredResponse = JSON.parse(codingRequiredResponse || "{}");
+                            } catch (error) {
+                                logger.error(`Error parsing codingRequiredResponse: ${error.message}, codingRequiredResponse was: ${codingRequiredResponse}`);
+                                parsedCodingRequiredResponse = {};
+                            }
+                            const { codingRequired } = parsedCodingRequiredResponse;
+                            if (codingRequired) {
+                                const { codingMessage, codingTask, codingTaskKeywords } = parsedCodingRequiredResponse;
+                                const message = typeof codingTask === 'string' 
+                                    ? codingTask 
+                                    : JSON.stringify(codingTask);
+                                const { contextId } = args;
+                                logger.info(`Sending task message coding agent: ${message}`);
+                                codeRequestId = await sendMessageToQueue({ message, contextId, keywords: codingTaskKeywords });
+
+                                toolCallbackId = codeRequestId;
+                                toolCallbackName = "coding";
+                                toolCallbackMessage = codingMessage;
+                                break;
+                            }
+                        }
+                        break;
+                    case "image":
+                        toolCallbackName = 'sys_generator_image';
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "vision":
+                    case "video":
+                    case "audio":
+                    case "pdf":
+                        if (visionContentPresent) {
+                            toolCallbackName = 'sys_generator_video_vision';
+                            toolCallbackMessage = toolMessage;
+                        }
+                        break;
+                    case "code":
+                    case "write":
+                        toolCallbackName = 'sys_generator_expert';
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "reason":
+                        toolCallbackName = 'sys_generator_reasoning';
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "search":
+                        toolCallbackName = 'sys_generator_results';
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "document":
+                        toolCallbackName = 'sys_generator_document';
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "clarify":
+                        toolCallbackName = null;
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    default:
+                        toolCallbackName = null;
+                        toolCallbackId = null;
+                        toolCallbackMessage = null;
+                        break;
+                }
+            }
+
+            if (toolCallbackMessage) {
+                if (args.stream) {
+                    await say(pathwayResolver.requestId, toolCallbackMessage || "One moment please.", 10);
+                    pathwayResolver.tool = JSON.stringify({ hideFromModel: false, search: false, title });  
+                    await callPathway('sys_entity_continue', { ...args, stream: true, model: styleModel, generatorPathway: toolCallbackName }, pathwayResolver);
+                    return "";
+                } else {
+                    pathwayResolver.tool = JSON.stringify({ 
+                        hideFromModel: toolCallbackName ? true : false, 
+                        toolCallbackName, 
+                        title,
+                        search: toolCallbackName === 'sys_generator_results' ? true : false,
+                        coding: toolCallbackName === 'coding' ? true : false,
+                        codeRequestId,
+                        toolCallbackId
+                    });
+                    return toolCallbackMessage || "One moment please.";
+                }
+            }
+
+            fetchChatResponsePromise = fetchChatResponsePromise || fetchChatResponse({ ...args }, pathwayResolver);
+            const chatResponse = await fetchChatResponsePromise;
+            pathwayResolver.tool = JSON.stringify({ search: false, title })
+            return args.stream ? "" : chatResponse;
+
+        } catch (e) {
+            pathwayResolver.logError(e);
+            fetchChatResponsePromise = fetchChatResponsePromise || fetchChatResponse({ ...args }, pathwayResolver);
+            const chatResponse = await fetchChatResponsePromise;
+            pathwayResolver.tool = JSON.stringify({ search: false, title });
+            return args.stream ? "" : chatResponse;
+        }
+    }
+};
+

package/pathways/system/entity/sys_generator_error.js

@@ -0,0 +1,20 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_DIRECTIVES}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\nThe user has requested information that you have already determined can be found in the indexes that you can search, and you were trying to search for it, but encountered the following error: {{{text}}}. Your response should be concise, fit the rest of the conversation, include detail appropriate for the technical level of the user if you can determine it, and be appropriate for the context. You cannot resolve this error.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        text: '',
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+}