@aj-archipelago/cortex 1.2.0 → 1.3.0

package/pathways/system/entity/sys_entity_start.js

@@ -0,0 +1,218 @@
+// sys_entity_start.js
+// Beginning of the rag workflow for Jarvis
+import { callPathway } from '../../../lib/pathwayTools.js';
+import logger from  '../../../lib/logger.js';
+import { chatArgsHasImageUrl } from  '../../../lib/util.js';
+import { QueueServiceClient } from '@azure/storage-queue';
+import entityConstants from './shared/sys_entity_constants.js';
+
+const TOKEN_RATIO = 0.75;
+
+const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
+let queueClient;
+
+if (connectionString) {
+  const queueName = process.env.AUTOGEN_MESSAGE_QUEUE || "autogen-message-queue";
+  const queueClientService = QueueServiceClient.fromConnectionString(connectionString);
+  queueClient = queueClientService.getQueueClient(queueName);
+} else {
+  logger.warn("Azure Storage connection string is not provided. Queue operations will be unavailable.");
+}
+
+async function sendMessageToQueue(data) {
+    try {
+        if(!queueClient){
+            logger.warn("Azure Storage connection string is not provided. Queue operations will be unavailable.");
+            return;
+        }
+        const encodedMessage = Buffer.from(JSON.stringify(data)).toString('base64');
+        const result = await queueClient.sendMessage(encodedMessage);
+        logger.info(`Message added to queue: ${JSON.stringify(result)}`);
+        return result.messageId;
+    } catch (error) {
+        logger.error("Error sending message:", error);
+    }
+}
+
+export default {
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    model: 'oai-gpt4o',
+    anthropicModel: 'claude-35-sonnet-vertex',
+    openAIModel: 'oai-gpt4o',
+    inputParameters: {
+        privateData: false,    
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        dataSources: ["mydata", "aja", "aje", "wires", "bing"],
+        language: "English",
+        aiName: "Jarvis",
+        aiMemorySelfModify: true,
+        aiStyle: "OpenAI",
+        title: ``,
+    },
+    timeout: 600,
+    tokenRatio: TOKEN_RATIO,
+    ...entityConstants,
+
+    executePathway: async ({args, resolver}) => {
+        let title = null;
+        let codeRequestId = null;
+
+        args = {
+            ...args,
+            ...entityConstants
+        };
+
+        // Limit the chat history to 20 messages to speed up processing
+        args.chatHistory = (args.messages || args.chatHistory).slice(-20);
+        const pathwayResolver = resolver;
+        const { anthropicModel, openAIModel } = pathwayResolver.pathway;
+
+        const styleModel = args.aiStyle === "Anthropic" ? anthropicModel : openAIModel;
+
+        // if the model has been overridden, make sure to use it
+        if (pathwayResolver.modelName) {
+            args.model = pathwayResolver.modelName;
+        }
+        
+        const fetchChatResponse = async (args) => {
+            const [chatResponse, chatTitleResponse] = await Promise.all([
+                callPathway('sys_generator_quick', {...args, model: styleModel}),
+                callPathway('chat_title', { ...args }),
+            ]);
+
+            title = chatTitleResponse;
+
+            return chatResponse;
+        };
+            
+        const { chatHistory } = args;
+
+        // start fetching the default response - we may need it later
+        const fetchChatResponsePromise = fetchChatResponse({ ...args });
+
+        const visionContentPresent = chatArgsHasImageUrl(args);
+
+        try {
+            // Get tool routing response
+            const toolRequiredResponse = await callPathway('sys_router_tool', { 
+                ...args,
+                chatHistory: chatHistory.slice(-4) 
+            });
+
+            // Asynchronously manage memory for this context
+            if (args.aiMemorySelfModify) {
+                callPathway('sys_memory_manager', {  ...args })    
+                .catch(error => logger.error(error?.message || "Error in sys_memory_manager pathway"));
+            }
+
+            const { toolRequired, toolMessage, toolFunction } = JSON.parse(toolRequiredResponse || '{}');
+            let toolCallbackName, toolCallbackId, toolCallbackMessage;
+
+            logger.info(`toolRequired: ${toolRequired}, toolFunction: ${toolFunction}`);
+
+            if (toolRequired && toolFunction) {
+                switch (toolFunction.toLowerCase()) {
+                    case "codeexecution":
+                        {
+                            const codingRequiredResponse = await callPathway('sys_router_code', { ...args });
+                            let parsedCodingRequiredResponse;
+                            try {
+                                parsedCodingRequiredResponse = JSON.parse(codingRequiredResponse || "{}");
+                            } catch (error) {
+                                logger.error(`Error parsing codingRequiredResponse: ${error.message}, codingRequiredResponse was: ${codingRequiredResponse}`);
+                                parsedCodingRequiredResponse = {};
+                            }
+                            const { codingRequired } = parsedCodingRequiredResponse;
+                            if (codingRequired) {
+                                const { codingMessage, codingTask, codingTaskKeywords } = parsedCodingRequiredResponse;
+                                const message = typeof codingTask === 'string' 
+                                    ? codingTask 
+                                    : JSON.stringify(codingTask);
+                                const { contextId } = args;
+                                logger.info(`Sending task message coding agent: ${message}`);
+                                codeRequestId = await sendMessageToQueue({ message, contextId, keywords: codingTaskKeywords });
+
+                                toolCallbackId = codeRequestId;
+                                toolCallbackName = "coding";
+                                toolCallbackMessage = codingMessage;
+                                break;
+                            }
+                        }
+                        break;
+                    case "image":
+                        toolCallbackName = 'sys_generator_image';
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "vision":
+                    case "video":
+                    case "audio":
+                    case "pdf":
+                        if (visionContentPresent) {
+                            toolCallbackName = 'sys_generator_video_vision';
+                            toolCallbackMessage = toolMessage;
+                        }
+                        break;
+                    case "code":
+                    case "write":
+                        toolCallbackName = 'sys_generator_expert';
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "reason":
+                        toolCallbackName = 'sys_generator_reasoning';
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "search":
+                        toolCallbackName = 'sys_generator_results';
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "document":
+                        toolCallbackName = 'sys_generator_document';
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    case "clarify":
+                        toolCallbackName = null;
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
+                    default:
+                        toolCallbackName = null;
+                        toolCallbackId = null;
+                        toolCallbackMessage = null;
+                        break;
+                }
+            }
+
+            if (toolCallbackMessage) {
+                pathwayResolver.tool = JSON.stringify({ 
+                    hideFromModel: toolCallbackName ? true : false, 
+                    toolCallbackName, 
+                    title,
+                    search: toolCallbackName === 'sys_generator_results' ? true : false,
+                    coding: toolCallbackName === 'coding' ? true : false,
+                    codeRequestId,
+                    toolCallbackId
+                });
+                return toolCallbackMessage || "One moment please.";
+            }
+
+            const chatResponse = await fetchChatResponsePromise;
+            pathwayResolver.tool = JSON.stringify({ search: false, title })
+            return chatResponse;
+
+        } catch (e) {
+            pathwayResolver.logError(e);
+            const chatResponse = await fetchChatResponsePromise;
+            pathwayResolver.tool = JSON.stringify({ search: false, title });
+            return chatResponse;
+        }
+    }
+};
+

package/pathways/system/entity/sys_generator_error.js

@@ -0,0 +1,20 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_DIRECTIVES}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\nThe user has requested information that you have already determined can be found in the indexes that you can search, and you were trying to search for it, but encountered the following error: {{{text}}}. Your response should be concise, fit the rest of the conversation, include detail appropriate for the technical level of the user if you can determine it, and be appropriate for the context. You cannot resolve this error.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        text: '',
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+}

package/pathways/system/entity/sys_generator_expert.js

@@ -0,0 +1,26 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "writing" });
+        return result;
+    }
+}

package/pathways/system/entity/sys_generator_image.js

@@ -0,0 +1,127 @@
+// sys_generator_image.js
+// Entity module that creates and shows images to the user
+import { callPathway } from '../../../lib/pathwayTools.js';
+import { Prompt } from '../../../server/prompt.js';
+import logger from '../../../lib/logger.js';
+import { getUniqueId } from '../../../lib/util.js';
+
+const TOKEN_RATIO = 1.0;
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        privateData: false,
+        useMemory: true,    
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        language: "English",
+        chatId: ``,
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    tokenRatio: TOKEN_RATIO,
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        const { chatHistory } = args;
+
+        let pathwayResolver = resolver;
+
+        const useMemory = args.useMemory || pathwayResolver.pathway.inputParameters.useMemory;
+
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: [
+                {
+                    "role": "system",
+                    "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}
+
+{{renderTemplate AI_DIRECTIVES}}
+
+Instructions: As part of a conversation with the user, you have been asked to create one or more images, photos, pictures, selfies, drawings, or other visual content for the user. You have already written the prompts and created the images - links to them are in the most recent tool calls in the chat history. You should display the images in a way that is most pleasing to the user. You can use markdown or HTML and img tags to display and format the images - the UI will render either. If there are no tool results, it means you didn't successfully create any images - in that case, don't show any images and tell the user you weren't able to create images.`
+                },
+                "{{chatHistory}}",
+            ]}),
+        ];
+
+        // function to add tool_calls to the chatHistory
+        const addToolCalls= (chatHistory, imagePrompt, toolCallId) => {
+            const toolCall = {
+                "role": "assistant",
+                "tool_calls": [
+                        {
+                            "id": toolCallId,
+                            "type": "function",
+                            "function": {
+                                "arguments": JSON.stringify(imagePrompt),
+                                "name": "generate_image"
+                            }
+                        }
+                    ]
+            };
+            chatHistory.push(toolCall);
+            return chatHistory;
+        }
+
+        // function to add tool_results to the chatHistory
+        const addToolResults = (chatHistory, imageResults, toolCallId) => {
+            const toolResult = {
+                "role": "tool",
+                "content": imageResults,
+                "tool_call_id": toolCallId
+            };
+            chatHistory.push(toolResult);
+            return chatHistory;
+        }
+
+        try {
+          
+            // figure out what the user wants us to do
+            const contextInfo = chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+            
+            const helper = await callPathway('sys_image_prompt_builder', { ...args, stream: false, useMemory, contextInfo });
+            logger.debug(`Image prompt builder response: ${helper}`);
+            const parsedHelper = JSON.parse(helper);
+            
+            //parsedHelper should always be an array of objects, but in case it's a single object, we'll wrap it in an array
+            const imagePrompts = Array.isArray(parsedHelper) ? parsedHelper : [parsedHelper];
+
+            //for each image prompt, create the images
+            const imageResults = await Promise.all(imagePrompts.map(async (imagePrompt) => {
+                const { prompt, numberResults, negativePrompt, renderText, draft } = imagePrompt;
+                if (!prompt) return null;
+                
+                let model = "replicate-flux-11-pro";
+                if (numberResults > 1 || draft) {
+                    model = "replicate-flux-1-schnell";
+                }
+                if (renderText) {
+                    return await callPathway('image_recraft', {...args, text: prompt });
+                } else {
+                    return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model });
+                }
+            })).then(results => results.filter(r => r !== null));
+
+            // add the tool_calls and tool_results to the chatHistory
+            imageResults.forEach((imageResult, index) => {
+                const toolCallId = getUniqueId();
+                addToolCalls(chatHistory, imagePrompts[index], toolCallId);
+                addToolResults(chatHistory, imageResult, toolCallId);
+            });
+            
+            const result = await runAllPrompts({ ...args });
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
+            return result;
+        } catch (e) {
+            pathwayResolver.logError(e.message ?? e);
+            return await callPathway('sys_generator_error', { ...args, text: e.message }, pathwayResolver);
+        }
+    }
+};

package/pathways/system/entity/sys_generator_quick.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../../../server/prompt.js';
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\nThe UI also has dedicated tabs to help with document translation (translate), article writing assistance including generating headlines, summaries and doing copy editing (write), video and audio transcription (transcribe), and programming and writing code (code). If the user asks about something related to a dedicated tab, you will tell them that the tab exists and the interface will give the user the option to swap to that tab.\n{{renderTemplate AI_EXPERTISE}}\nYou have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation.\nNever pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+        model: "oai-gpt4o",
+    },
+    useInputChunking: false,
+    enableDuplicateRequests: false
+}

package/pathways/system/entity/sys_generator_reasoning.js

@@ -0,0 +1,27 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nUse all of the information in your memory and the chat history to reason about the user's request and provide a response. Often this information will be more current than your knowledge cutoff.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-o1-mini',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "reasoning" });
+        return result;
+    }
+        
+}