@aj-archipelago/cortex 1.3.35 → 1.3.36

package/pathways/system/entity/tools/sys_tool_callmodel.js

@@ -0,0 +1,62 @@
+// sys_tool_callmodel.js
+// Entity tool that calls a model to get a response
+import { Prompt } from '../../../../server/prompt.js';
+import logger from '../../../../lib/logger.js';
+
+export default {
+    prompt: [
+        new Prompt({
+            messages: [
+                { "role": "system", "content": "{{{systemPrompt}}}" },
+                { "role": "user", "content": "{{{userPrompt}}}" }
+            ]
+        }),
+    ],
+
+    inputParameters: {
+        userPrompt: "",
+        systemPrompt: "",
+        model: "oai-gpt41"
+    },
+    
+    toolDefinition:     {
+        type: "function",
+        icon: "🤖",
+        function: {
+            name: "CallModel",
+            description: "Use when you need to call an AI model to get a response. This is typically used to perform some sort of LLM analysis (translate, summarize, ask questions about content, etc.), but can literally do anything you need. You can use this to call any model you have access to and perform any task.",
+            parameters: {
+                type: "object",
+                properties: {
+                    systemPrompt: {
+                        type: "string",
+                        description: "The system prompt to send to the model to set up the context for what you want the model to do."
+                    },
+                    userPrompt: {
+                        type: "string",
+                        description: "The complete prompt to send as a user message to the model instructing the model to perform the task you need."
+                    },
+                    model: {
+                        type: "string",
+                        description: "The model to use. You currently have the following models available to call: oai-gpt4o, oai-gpt41, oai-o3, oai-o3-mini, claude-35-sonnet-vertex, gemini-flash-20-vision, gemini-pro-25-vision."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["systemPrompt", "userPrompt", "model", "userMessage"]
+            }
+        }
+    },
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        try {
+            const result = await runAllPrompts({ ...args });
+            return result;
+        } catch (error) {
+            logger.error(error);
+            return "Error calling model: " + error.message;
+        }
+    }
+}

package/pathways/system/entity/tools/sys_tool_coding.js

@@ -0,0 +1,53 @@
+// sys_tool_coding.js
+// Entity tool that provides advanced coding and programming capabilities
+
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides advanced coding and programming capabilities. You excel at writing, reviewing, and explaining code across various programming languages. You can help with code generation, debugging, optimization, and best practices. Think carefully about the latest request and provide a detailed, well thought out, carefully reviewed response.\n{{renderTemplate AI_DATETIME}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 100000,
+    model: 'oai-o3',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: [{
+        type: "function",
+        icon: "💻",
+        function: {
+            name: "Code",
+            description: "Engage for any programming-related tasks, including writing, modifying, reviewing, or explaining code.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    }],
+    
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        let result = await runAllPrompts({ ...args, stream: false });        
+        resolver.tool = JSON.stringify({ toolUsed: "coding" });          
+        return result;
+    }
+} 

package/pathways/system/entity/tools/sys_tool_codingagent.js

@@ -0,0 +1,100 @@
+// sys_tool_codingagent.js
+// Entity tool that provides code execution capabilities through a queue-based system
+
+import { QueueServiceClient } from '@azure/storage-queue';
+import logger from '../../../../lib/logger.js';
+
+const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
+let queueClient;
+
+if (connectionString) {
+  const queueName = process.env.AUTOGEN_MESSAGE_QUEUE || "autogen-message-queue";
+  const queueClientService = QueueServiceClient.fromConnectionString(connectionString);
+  queueClient = queueClientService.getQueueClient(queueName);
+} else {
+  logger.warn("Azure Storage connection string is not provided. Queue operations will be unavailable.");
+}
+
+async function sendMessageToQueue(data) {
+    try {
+        if(!queueClient){
+            logger.warn("Azure Storage connection string is not provided. Queue operations will be unavailable.");
+            return;
+        }
+        const encodedMessage = Buffer.from(JSON.stringify(data)).toString('base64');
+        const result = await queueClient.sendMessage(encodedMessage);
+        logger.info(`Message added to queue: ${JSON.stringify(result)}`);
+        return result.messageId;
+    } catch (error) {
+        logger.error("Error sending message:", error);
+        throw error;
+    }
+}
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 100000,
+    model: 'oai-gpt41',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: [{
+        type: "function",
+        icon: "🤖",
+        function: {
+            name: "CodeExecution",
+            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks.",
+            parameters: {
+                type: "object",
+                properties: {
+                    codingTask: {
+                        type: "string",
+                        description: "Detailed task description for the coding agent. Include all necessary information as this is the only message the coding agent receives. Let the agent decide how to solve it without making assumptions about its capabilities."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message to notify the user that a coding task is being handled"
+                    },
+                    codingTaskKeywords: {
+                        type: "string",
+                        description: "Keywords for Azure Cognitive Search to help the coding agent find relevant code snippets"
+                    }
+                },
+                required: ["codingTask", "userMessage", "codingTaskKeywords"]
+            }
+        }
+    }],
+    
+    executePathway: async ({args, resolver}) => {
+        try {
+            const { codingTask, userMessage, codingTaskKeywords } = args;
+            const { contextId } = args;
+
+            // Send the task to the queue
+            const codeRequestId = await sendMessageToQueue({ 
+                message: codingTask, 
+                contextId, 
+                keywords: codingTaskKeywords 
+            });
+
+            // Set the tool response
+            resolver.tool = JSON.stringify({ 
+                toolUsed: "coding",
+                codeRequestId,
+                toolCallbackName: "coding",
+                toolCallbackId: codeRequestId,
+                toolCallbackMessage: userMessage
+            });
+
+            return userMessage || "I've started working on your coding task. I'll let you know when it's complete.";
+        } catch (error) {
+            logger.error("Error in coding agent tool:", error);
+            throw error;
+        }
+    }
+}; 

package/pathways/system/entity/tools/sys_tool_cognitive_search.js

@@ -0,0 +1,231 @@
+// sys_tool_cognitive_search.js
+// Tool pathway that handles cognitive search across various indexes
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { Prompt } from '../../../../server/prompt.js';
+import logger from '../../../../lib/logger.js';
+import { getSearchResultId } from '../../../../lib/util.js';
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        text: '',
+        filter: '',
+        top: 50,
+        titleOnly: false,
+        stream: false,
+        indexName: ''
+    },
+    timeout: 300,
+    toolDefinition: [
+        {
+            type: "function",
+            icon: "📂",
+            function: {
+                name: "SearchPersonal",
+                description: "Search through the user's personal documents and uploaded files. Use this for finding information in user-provided content.",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        text: {
+                            type: "string",
+                            description: "The search query to find relevant content in personal documents. Can be a specific phrase or '*' for all documents."
+                        },
+                        filter: {
+                            type: "string",
+                            description: "Optional OData filter expression for date filtering (e.g. 'date ge 2024-02-22T00:00:00Z')"
+                        },
+                        top: {
+                            type: "integer",
+                            description: "Number of results to return (default is 50)"
+                        },
+                        titleOnly: {
+                            type: "boolean",
+                            description: "If true, only return document titles without content"
+                        },
+                        userMessage: {
+                            type: "string",
+                            description: "A user-friendly message that describes what you're doing with this tool"
+                        }
+                    },
+                    required: ["text", "userMessage"]
+                }
+            }
+        },
+        {
+            type: "function",
+            function: {
+                name: "SearchAJA",
+                icon: "📰",
+                description: "Search through Al Jazeera Arabic news articles. Use this for finding Arabic news content.",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        text: {
+                            type: "string",
+                            description: "The search query in Arabic to find relevant news articles. Can be a specific phrase or '*' for all articles."
+                        },
+                        filter: {
+                            type: "string",
+                            description: "Optional OData filter expression for date filtering (e.g. 'date ge 2024-02-22T00:00:00Z')"
+                        },
+                        top: {
+                            type: "integer",
+                            description: "Number of results to return (default is 50)"
+                        },
+                        titleOnly: {
+                            type: "boolean",
+                            description: "If true, only return article titles without content"
+                        },
+                        userMessage: {
+                            type: "string",
+                            description: "A user-friendly message that describes what you're doing with this tool"
+                        }
+                    },
+                    required: ["text", "userMessage"]
+                }
+            }
+        },
+        {
+            type: "function",
+            icon: "📰",
+            function: {
+                name: "SearchAJE",
+                description: "Search through Al Jazeera English news articles. Use this for finding English news content.",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        text: {
+                            type: "string",
+                            description: "The search query in English to find relevant news articles. Can be a specific phrase or '*' for all articles."
+                        },
+                        filter: {
+                            type: "string",
+                            description: "Optional OData filter expression for date filtering (e.g. 'date ge 2024-02-22T00:00:00Z')"
+                        },
+                        top: {
+                            type: "integer",
+                            description: "Number of results to return (default is 50)"
+                        },
+                        titleOnly: {
+                            type: "boolean",
+                            description: "If true, only return article titles without content"
+                        },
+                        userMessage: {
+                            type: "string",
+                            description: "A user-friendly message that describes what you're doing with this tool"
+                        }
+                    },
+                    required: ["text", "userMessage"]
+                }
+            }
+        },
+        {
+            type: "function",
+            icon: "⚡️",
+            function: {
+                name: "SearchWires",
+                description: "Search through news wires from all sources. Use this for finding the latest news and articles.",
+                parameters: {
+                    type: "object",
+                    properties: {
+                        text: {
+                            type: "string",
+                            description: "The search query to find relevant news wires. Can be a specific phrase or '*' for all wires."
+                        },
+                        filter: {
+                            type: "string",
+                            description: "Optional OData filter expression for date filtering (e.g. 'date ge 2024-02-22T00:00:00Z')"
+                        },
+                        top: {
+                            type: "integer",
+                            description: "Number of results to return (default is 50)"
+                        },
+                        titleOnly: {
+                            type: "boolean",
+                            description: "If true, only return wire titles without content"
+                        },
+                        userMessage: {
+                            type: "string",
+                            description: "A user-friendly message that describes what you're doing with this tool"
+                        }
+                    },
+                    required: ["text", "userMessage"]
+                }
+            }
+        }
+    ],
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const { text, filter, top, titleOnly, stream, dataSources, indexName, semanticConfiguration } = args;
+
+        // Map tool names to index names
+        const toolToIndex = {
+            'searchpersonal': 'indexcortex',
+            'searchaja': 'indexucmsaja',
+            'searchaje': 'indexucmsaje',
+            'searchwires': 'indexwires'
+        };
+
+        // Helper function to remove vector fields from search results
+        const removeVectorFields = (result) => {
+            const { text_vector, image_vector, ...cleanResult } = result;
+            return cleanResult;
+        };
+
+        // Get the tool name from the function call
+        const toolName = args.toolFunction;
+        const toolIndexName = indexName || toolToIndex[toolName];
+
+        if (!toolName || !toolIndexName) {
+            throw new Error(`Invalid tool name: ${toolName}. Search not allowed.`);
+        }
+
+        try {
+            // Call the cognitive search pathway
+            const response = await callPathway('cognitive_search', {
+                ...args,
+                text,
+                filter,
+                top: top || 50,
+                titleOnly: titleOnly || false,
+                indexName: toolIndexName,
+                semanticConfiguration,
+                stream: stream || false
+            });
+
+            const parsedResponse = JSON.parse(response);
+
+            const combinedResults = [];
+
+            if (parsedResponse.value && Array.isArray(parsedResponse.value)) {
+                // Filter out vector fields from each result before adding to combinedResults
+                combinedResults.push(...parsedResponse.value.map(result => ({
+                    ...removeVectorFields(result),
+                    searchResultId: getSearchResultId()
+                })));
+            }
+            // Extract semantic answers
+            const answers = parsedResponse["@search.answers"];
+            if (answers && Array.isArray(answers)) {
+                const formattedAnswers = answers.map(ans => ({
+                    // Create a pseudo-document structure for answers
+                    searchResultId: getSearchResultId(),
+                    title: "", // no title for answers
+                    content: ans.text || "", // Use text as content
+                    key: ans.key, // Keep the key if needed later
+                    score: ans.score, // Keep score
+                    source_type: 'answer' // Add a type identifier
+                    // url: null - Answers don't have URLs
+                }));
+                combinedResults.push(...formattedAnswers);
+            }
+
+            return JSON.stringify({ _type: "SearchResponse", value: combinedResults });
+        } catch (e) {
+            logger.error(`Error in cognitive search for index ${indexName}: ${e}`);
+            throw e;
+        }
+    }
+}; 

package/pathways/system/entity/tools/sys_tool_image.js

@@ -0,0 +1,57 @@
+// sys_tool_image.js
+// Entity tool that creates images for the entity to show to the user
+import { callPathway } from '../../../../lib/pathwayTools.js';
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    toolDefinition: {
+        type: "function",
+        icon: "🎨",
+        function: {
+            name: "Image",
+            description: "Use when asked to create, generate, or revise visual content. Any time the user asks you for a picture, a selfie, artwork, a drawing or if you want to illustrate something for the user, you can use this tool to generate any sort of image from cartoon to photo realistic.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "A very detailed prompt describing the image you want to create. You should be very specific - explaining subject matter, style, and details about the image including things like camera angle, lens types, lighting, photographic techniques, etc. Any details you can provide to the image creation engine will help it create the most accurate and useful images. The more detailed and descriptive the prompt, the better the result."
+                    },
+                    renderText: {
+                        type: "boolean",
+                        description: "Set to true if the image should be optimized to show correct text. This is useful when the user asks for a picture of something that includes specific text as it invokes a different image generation model that is optimized for including text."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "renderText", "userMessage"]
+            }
+        }
+    },
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const pathwayResolver = resolver;
+
+        try {   
+            let model = "replicate-flux-11-pro";
+            let prompt = args.detailedInstructions;
+            let numberResults = args.numberResults || 1;
+            let negativePrompt = args.negativePrompt || "";
+
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
+            return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model, stream: false });
+
+        } catch (e) {
+            pathwayResolver.logError(e.message ?? e);
+            return await callPathway('sys_generator_error', { ...args, text: e.message }, pathwayResolver);
+        }
+    }
+};

package/pathways/system/entity/tools/sys_tool_readfile.js

@@ -0,0 +1,119 @@
+// sys_tool_readfile.js
+// Entity tool that reads one or more files and answers questions about them
+
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that can view, hear, and understand files of all sorts (images, videos, audio, pdfs, text, etc.) - you provide the capability to view and analyze files that the user provides.\nThe user has provided you with one or more files in this conversation - you should consider them for context when you respond.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.\n{{renderTemplate AI_DATETIME}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 8192,
+    model: 'gemini-flash-20-vision',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: [{
+        type: "function",
+        icon: "📄",
+        function: {
+            name: "PDF",
+            description: "Use specifically for analyzing and answering questions about PDF file content.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    },
+    {
+        type: "function",
+        icon: "📝",
+        function: {
+            name: "Text",
+            description: "Use specifically for analyzing and answering questions about text files.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
+                        },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                        }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    },  
+    {
+        type: "function",
+        icon: "🖼️",
+        function: {
+            name: "Vision",
+            description: "Use specifically for analyzing and answering questions about image files (jpg, gif, bmp, png, etc).",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    },
+    {
+        type: "function",
+        icon: "🎥",
+        function: {
+            name: "Video",
+            description: "Use specifically for analyzing and answering questions about video or audio file content. You MUST use this tool to look at video or audio files.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    }],
+    
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "vision" });
+        return result;
+    }
+}