@aj-archipelago/cortex 1.3.35 → 1.3.36

package/pathways/system/entity/sys_entity_agent.js

@@ -0,0 +1,289 @@
+// sys_entity_agent.js
+// Agentic extension of the entity system that uses OpenAI's tool calling API
+import { callPathway, callTool, say } from '../../../lib/pathwayTools.js';
+import logger from '../../../lib/logger.js';
+import { config } from '../../../config.js';
+import { chatArgsHasImageUrl, removeOldImageAndFileContent } from '../../../lib/util.js';
+import { insertToolCallAndResults } from './memory/shared/sys_memory_helpers.js';
+import { Prompt } from '../../../server/prompt.js';
+import { getToolsForEntity, loadEntityConfig } from './tools/shared/sys_entity_tools.js';
+
+export default {
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    useSingleTokenStream: false,
+    inputParameters: {
+        privateData: false,    
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        dataSources: ["mydata", "aja", "aje", "wires", "bing"],
+        language: "English",
+        aiName: "Jarvis",
+        aiMemorySelfModify: true,
+        aiStyle: "OpenAI",
+        title: ``,
+        messages: [],
+        voiceResponse: false,
+        codeRequestId: ``,
+        skipCallbackMessage: false,
+        entityId: ``,
+        model: 'oai-gpt41'
+    },
+    timeout: 600,
+
+    toolCallback: async (args, message, resolver) => {
+        const { tool_calls } = message;
+        const pathwayResolver = resolver;
+        const { entityTools, entityToolsOpenAiFormat } = args;
+        
+        // Make a deep copy of the initial chat history
+        const initialMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
+
+        if (tool_calls) {
+            // Execute tool calls in parallel but with isolated message histories
+            const toolResults = await Promise.all(tool_calls.map(async (toolCall) => {
+                try {
+                    if (!toolCall?.function?.arguments) {
+                        throw new Error('Invalid tool call structure: missing function arguments');
+                    }
+
+                    const toolArgs = JSON.parse(toolCall.function.arguments);
+                    const toolFunction = toolCall.function.name.toLowerCase();
+                    
+                    // Create an isolated copy of messages for this tool
+                    const toolMessages = JSON.parse(JSON.stringify(initialMessages));
+                    
+                    // Get the tool definition to check for icon
+                    const toolDefinition = entityTools[toolFunction]?.definition;
+                    const toolIcon = toolDefinition?.icon || '🛠️';
+                    
+                    // Report status to the user
+                    const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name} - ${JSON.stringify(toolArgs)}`;
+                    const messageWithIcon = toolIcon ? `${toolIcon}  ${toolUserMessage}` : toolUserMessage;
+                    await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
+
+                    if (toolArgs.detailedInstructions) {
+                        toolMessages.push({role: "user", content: toolArgs.detailedInstructions});
+                    }
+
+                    // Add the tool call to the isolated message history
+                    toolMessages.push({
+                        role: "assistant",
+                        content: "",
+                        tool_calls: [{
+                            id: toolCall.id,
+                            type: "function",
+                            function: {
+                                name: toolCall.function.name,
+                                arguments: JSON.stringify(toolArgs)
+                            }
+                        }]
+                    });
+
+                    const toolResult = await callTool(toolFunction, {
+                        ...args,
+                        ...toolArgs,
+                        toolFunction,
+                        chatHistory: toolMessages,
+                        stream: false
+                    }, entityTools, pathwayResolver);
+
+                    // Add the tool result to the isolated message history
+                    let toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult);
+
+                    toolMessages.push({
+                        role: "tool",
+                        tool_call_id: toolCall.id,
+                        name: toolCall.function.name,
+                        content: toolResultContent
+                    });
+
+                    return { 
+                        success: true, 
+                        result: toolResult,
+                        toolCall,
+                        toolArgs,
+                        toolFunction,
+                        messages: toolMessages
+                    };
+                } catch (error) {
+                    logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
+                    
+                    // Create error message history
+                    const errorMessages = JSON.parse(JSON.stringify(initialMessages));
+                    errorMessages.push({
+                        role: "assistant",
+                        content: "",
+                        tool_calls: [{
+                            id: toolCall.id,
+                            type: "function",
+                            function: {
+                                name: toolCall.function.name,
+                                arguments: JSON.stringify(toolCall.function.arguments)
+                            }
+                        }]
+                    });
+                    errorMessages.push({
+                        role: "tool",
+                        tool_call_id: toolCall.id,
+                        name: toolCall.function.name,
+                        content: `Error: ${error.message}`
+                    });
+
+                    return { 
+                        success: false, 
+                        error: error.message,
+                        toolCall,
+                        toolArgs: toolCall?.function?.arguments ? JSON.parse(toolCall.function.arguments) : {},
+                        toolFunction: toolCall?.function?.name?.toLowerCase() || 'unknown',
+                        messages: errorMessages
+                    };
+                }
+            }));
+
+            // Merge all message histories in order
+            let finalMessages = JSON.parse(JSON.stringify(initialMessages));
+            for (const result of toolResults) {
+                try {
+                    if (!result?.messages) {
+                        logger.error('Invalid tool result structure, skipping message history update');
+                        continue;
+                    }
+
+                    // Add only the new messages from this tool's history
+                    const newMessages = result.messages.slice(initialMessages.length);
+                    finalMessages.push(...newMessages);
+                } catch (error) {
+                    logger.error(`Error merging message history for tool result: ${error.message}`);
+                }
+            }
+
+            // Check if any tool calls failed
+            const failedTools = toolResults.filter(result => !result.success);
+            if (failedTools.length > 0) {
+                logger.warn(`Some tool calls failed: ${failedTools.map(t => t.error).join(', ')}`);
+            }
+
+            args.chatHistory = finalMessages;
+
+            await pathwayResolver.promptAndParse({
+                ...args,
+                tools: entityToolsOpenAiFormat,
+                tool_choice: "auto",
+                stream: true
+            });
+        }
+    },
+  
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        let pathwayResolver = resolver;
+
+        // Load input parameters and information into args
+        const { entityId, voiceResponse, aiMemorySelfModify } = { ...pathwayResolver.pathway.inputParameters, ...args };
+        
+        const entityConfig = loadEntityConfig(entityId);
+        const { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
+        const { useMemory: entityUseMemory = true, name: entityName, instructions: entityInstructions } = entityConfig || {};
+
+        if (entityId && entityName) {
+            args.aiName = entityName;
+        }
+        
+        args = {
+            ...args,
+            ...config.get('entityConstants'),
+            entityId,
+            entityTools,
+            entityToolsOpenAiFormat,
+            entityUseMemory,
+            entityInstructions,
+            voiceResponse,
+            aiMemorySelfModify
+        };
+
+        pathwayResolver.args = {...args};
+
+        const memoryTemplates = entityUseMemory ? 
+            `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n\n` : '';
+
+        const instructionTemplates = entityInstructions ? (entityInstructions + '\n\n') : `{{renderTemplate AI_EXPERTISE}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n`;
+
+        const promptMessages = [
+            {"role": "system", "content": `${memoryTemplates}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n{{renderTemplate AI_DATETIME}}`},
+            "{{chatHistory}}",
+        ];
+
+        pathwayResolver.pathwayPrompt = [
+            new Prompt({ messages: promptMessages }),
+        ];
+
+        // if the model has been overridden, make sure to use it
+        if (pathwayResolver.modelName) {
+            pathwayResolver.args.model = pathwayResolver.modelName;
+        }
+
+        // set the style model if applicable
+        const { aiStyle, AI_STYLE_ANTHROPIC, AI_STYLE_OPENAI } = args;
+        const styleModel = aiStyle === "Anthropic" ? AI_STYLE_ANTHROPIC : AI_STYLE_OPENAI;
+
+        // Initialize chat history if needed
+        if (!args.chatHistory || args.chatHistory.length === 0) {
+            args.chatHistory = [];
+        }
+
+        // Limit the chat history to 20 messages to speed up processing
+        if (args.messages && args.messages.length > 0) {
+            args.chatHistory = args.messages.slice(-20);
+        } else {
+            args.chatHistory = args.chatHistory.slice(-20);
+        }
+
+        // remove old image and file content
+        const visionContentPresent = chatArgsHasImageUrl(args);
+        visionContentPresent && (args.chatHistory = removeOldImageAndFileContent(args.chatHistory));
+
+        // truncate the chat history in case there is really long content
+        const truncatedChatHistory = resolver.modelExecutor.plugin.truncateMessagesToTargetLength(args.chatHistory, null, 1000);
+
+        // Add the memory context to the chat history if applicable
+        if (truncatedChatHistory.length > 1 && entityUseMemory) {
+            const memoryContext = await callPathway('sys_read_memory', { ...args, chatHistory: truncatedChatHistory, section: 'memoryContext', priority: 0, recentHours: 0, stream: false }, resolver);
+            if (memoryContext) {
+                insertToolCallAndResults(args.chatHistory, "Load general memory context information", "LoadMemoryContext", memoryContext);
+            }
+        }
+        
+        // Asynchronously manage memory for this context
+        if (args.aiMemorySelfModify && entityUseMemory) {
+            callPathway('sys_memory_manager', {  ...args, chatHistory: truncatedChatHistory, stream: false })    
+            .catch(error => logger.error(error?.message || "Error in sys_memory_manager pathway"));
+        }
+
+        try {
+            let currentMessages = JSON.parse(JSON.stringify(args.chatHistory));
+
+            // Run the initial prompt with streaming
+            const response = await runAllPrompts({
+                ...args,
+                chatHistory: currentMessages,
+                tools: entityToolsOpenAiFormat,
+                tool_choice: "auto",
+                stream: true
+            });
+
+
+            // Return the final response
+            return response;
+
+        } catch (e) {
+            resolver.logError(e);
+            const chatResponse = await callPathway('sys_generator_quick', {...args, model: styleModel, stream: false}, resolver);
+            resolver.tool = JSON.stringify({ search: false, title: args.title });
+            return chatResponse;
+        }
+    }
+}; 

package/pathways/system/entity/sys_generator_memory.js

@@ -10,7 +10,7 @@ export default {
         aiName: "Jarvis",
         language: "English",
     },
-    model: 'oai-gpt4o',
+    model: 'oai-gpt41-mini',
     useInputChunking: false,
     enableDuplicateRequests: false,
     executePathway: async ({args, resolver}) => {

package/pathways/system/entity/sys_generator_results.js

@@ -25,7 +25,7 @@ export default {
         language: "English",
         chatId: ``,
         dataSources: [""],
-        model: 'oai-gpt4o',
+        model: 'oai-gpt41-mini',
     },
     timeout: 300,
 

package/pathways/system/entity/sys_get_entities.js

@@ -0,0 +1,19 @@
+// sys_get_entities.js
+// Pathway to get list of available entities with their tools
+
+import { getAvailableEntities } from './tools/shared/sys_entity_tools.js';
+
+export default {
+    prompt: [],
+    inputParameters: {},
+    executePathway: async ({ args }) => {
+        try {
+            const entities = getAvailableEntities();
+            return JSON.stringify(entities);
+        } catch (error) {
+            return JSON.stringify(error);
+        }
+    },
+    json: true, // We want JSON output
+    manageTokenLength: false, // No need to manage token length for this simple operation
+}; 

package/pathways/system/entity/tools/shared/sys_entity_tools.js

@@ -0,0 +1,150 @@
+// sys_entity_tools.js
+// Shared tool definitions that can be used by any entity
+import { config } from '../../../../../config.js';
+import logger from '../../../../../lib/logger.js';
+
+export const CUSTOM_TOOLS = {};
+
+// Helper function to get tools for a specific entity
+export const getToolsForEntity = (entityConfig) => {
+    // Get system tools from config
+    const systemTools = config.get('entityTools') || {};
+    
+    // Convert all tool names to lowercase in system tools
+    const normalizedSystemTools = Object.fromEntries(
+        Object.entries(systemTools).map(([key, value]) => [key.toLowerCase(), value])
+    );
+    
+    // Convert custom tools to lowercase if they exist
+    const normalizedCustomTools = entityConfig?.customTools ? 
+        Object.fromEntries(
+            Object.entries(entityConfig.customTools).map(([key, value]) => [key.toLowerCase(), value])
+        ) : {};
+    
+    // Convert CUSTOM_TOOLS to lowercase
+    const normalizedCUSTOM_TOOLS = Object.fromEntries(
+        Object.entries(CUSTOM_TOOLS).map(([key, value]) => [key.toLowerCase(), value])
+    );
+    
+    // Merge system tools with custom tools (custom tools override system tools)
+    const allTools = { ...normalizedSystemTools, ...normalizedCustomTools, ...normalizedCUSTOM_TOOLS };
+    
+    // If no tools property specified or array contains *, return all tools
+    if (!entityConfig?.tools || entityConfig.tools.includes('*')) {
+        return {
+            entityTools: allTools,
+            entityToolsOpenAiFormat: Object.values(allTools).map(tool => {
+                const { icon, pathwayParams, ...definitionWithoutExtras } = tool.definition;
+                return definitionWithoutExtras;
+            })
+        };
+    }
+
+    // Get the list of tool names for this entity and convert to lowercase for case-insensitive comparison
+    const entityToolNames = entityConfig.tools.map(name => name.toLowerCase());
+    
+    // Add custom tools to the list of allowed tools if they exist
+    if (entityConfig.customTools) {
+        Object.keys(entityConfig.customTools).forEach(toolName => {
+            if (!entityToolNames.includes(toolName.toLowerCase())) {
+                entityToolNames.push(toolName.toLowerCase());
+            }
+        });
+    }
+    
+    // Filter the tools to only include those specified for this entity
+    const filteredTools = Object.fromEntries(
+        Object.entries(allTools).filter(([toolName]) => 
+            entityToolNames.includes(toolName.toLowerCase())
+        )
+    );
+
+    return {
+        entityTools: filteredTools,
+        entityToolsOpenAiFormat: Object.values(filteredTools).map(tool => {
+            const { icon, pathwayParams, ...definitionWithoutExtras } = tool.definition;
+            return definitionWithoutExtras;
+        })
+    };
+};
+
+// Load entity configurations
+export const loadEntityConfig = (entityId) => {
+    try {
+        entityId = entityId.toLowerCase();
+        
+        const entityConfig = config.get('entityConfig');
+        if (!entityConfig) {
+            logger.warn('No entity config found in config');
+            return null;
+        }
+
+        // Handle both array and object formats
+        const configArray = Array.isArray(entityConfig) ? entityConfig : Object.entries(entityConfig).map(([id, config]) => ({
+            id,
+            ...config
+        }));
+
+        // If entityId is provided, look for that specific entity
+        if (entityId) {
+            const entity = configArray.find(e => e.id === entityId);
+            if (entity) {
+                return entity;
+            }
+            logger.warn(`Entity ${entityId} not found in config`);
+        }
+
+        // If no entityId or entity not found, look for default entity
+        const defaultEntity = configArray.find(e => e.isDefault === true);
+        if (defaultEntity) {
+            return defaultEntity;
+        }
+
+        // If no default entity found, return the first entity
+        if (configArray.length > 0) {
+            return configArray[0];
+        }
+
+        return null;
+    } catch (error) {
+        logger.error(`Error loading entity config: ${error.message}`);
+        return null;
+    }
+};
+
+/**
+ * Fetches the list of available entities with their descriptions and active tools
+ * @returns {Array} Array of objects containing entity information and their active tools
+ */
+export const getAvailableEntities = () => {
+    try {
+        const entityConfig = config.get('entityConfig');
+        if (!entityConfig) {
+            logger.warn('No entity config found in config');
+            return [];
+        }
+
+        // Handle both array and object formats
+        const configArray = Array.isArray(entityConfig) ? entityConfig : Object.entries(entityConfig).map(([id, config]) => ({
+            id,
+            ...config
+        }));
+
+        return configArray.map(entity => {
+            const { entityTools } = getToolsForEntity(entity);
+            return {
+                id: entity.id,
+                name: entity.name || entity.id,
+                description: entity.description || '',
+                isDefault: entity.isDefault || false,
+                activeTools: Object.keys(entityTools).map(toolName => ({
+                    name: toolName,
+                    description: entityTools[toolName].definition?.function?.description || ''
+                }))
+            };
+        });
+    } catch (error) {
+        logger.error(`Error fetching available entities: ${error.message}`);
+        return [];
+    }
+};

package/pathways/system/entity/tools/sys_tool_bing_search.js

@@ -0,0 +1,147 @@
+// sys_tool_bing_search.js
+// Tool pathway that handles Bing web search functionality
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import logger from '../../../../lib/logger.js';
+import { config } from '../../../../config.js';
+import { getSearchResultId } from '../../../../lib/util.js';
+
+export default {
+    prompt: [],
+    timeout: 300,
+    toolDefinition: { 
+        type: "function",
+        icon: "🌐",
+        function: {
+            name: "InternetSearch",
+            description: "This tool allows you to use the Bing search api to search the internet and more. Use this for current events, news, fact-checking, and information requiring citation.",
+            parameters: {
+                type: "object",
+                properties: {
+                    q: {
+                        type: "string",
+                        description: "The complete query to pass to Azure Bing search. You can use advanced search operators in your query: + to require terms, \" \" for exact phrases, () for grouping, AND/& for requiring all terms, NOT/- to exclude terms, OR/| for either term. For example: '+(exact phrase) AND term1 -term2'"
+                    },
+                    freshness: {
+                        type: "string",
+                        description: "Filter results by freshness (when the content was first encountered by the search engine). Only use this if you need to be very specific as it may exclude many relevant results. Can be 'day', 'week', 'month', or a date range 'YYYY-MM-DD..YYYY-MM-DD'"
+                    },
+                    count: {
+                        type: "integer",
+                        description: "Number of webpages to return (default is 10)"
+                    },
+                    safeSearch: {
+                        type: "string",
+                        description: "Filter adult content. Can be 'Off', 'Moderate' (default), or 'Strict'"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["q", "userMessage"]
+            }
+        }
+    },
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        // Check if Bing API key is available
+        const bingAvailable = !!config.getEnv()["AZURE_BING_KEY"];
+        if (!bingAvailable) {
+            throw new Error("Bing search is not available - missing API key");
+        }
+
+        try {
+            // Call the Bing search pathway
+            const response = await callPathway('bing', { 
+                ...args
+            });
+
+            const parsedResponse = JSON.parse(response);
+            const results = [];
+
+            // Process web pages
+            if (parsedResponse.webPages && parsedResponse.webPages.value) {
+                results.push(...parsedResponse.webPages.value.map(({ name, url, snippet }) => ({
+                    searchResultId: getSearchResultId(),
+                    title: name,
+                    url,
+                    content: snippet
+                })));
+            }
+
+            // Process computation results
+            if (parsedResponse.computation) {
+                results.push({
+                    searchResultId: getSearchResultId(),
+                    title: "Computation Result",
+                    content: `Expression: ${parsedResponse.computation.expression}, Value: ${parsedResponse.computation.value}`
+                });
+            }
+
+            // Process entities
+            if (parsedResponse.entities && parsedResponse.entities.value) {
+                results.push(...parsedResponse.entities.value.map(entity => ({
+                    searchResultId: getSearchResultId(),
+                    title: entity.name,
+                    content: entity.description,
+                    url: entity.webSearchUrl
+                })));
+            }
+
+            // Process news
+            if (parsedResponse.news && parsedResponse.news.value) {
+                results.push(...parsedResponse.news.value.map(news => ({
+                    searchResultId: getSearchResultId(),
+                    title: news.name,
+                    content: news.description,
+                    url: news.url
+                })));
+            }
+
+            // Process videos
+            if (parsedResponse.videos && parsedResponse.videos.value) {
+                results.push(...parsedResponse.videos.value.map(video => ({
+                    searchResultId: getSearchResultId(),
+                    title: video.name,
+                    content: video.description,
+                    url: video.contentUrl
+                })));
+            }
+
+            // Process places
+            if (parsedResponse.places && parsedResponse.places.value) {
+                results.push(...parsedResponse.places.value.map(place => ({
+                    searchResultId: getSearchResultId(),
+                    title: place.name,
+                    content: `Address: ${place.address.addressLocality}, ${place.address.addressRegion}, ${place.address.addressCountry}`,
+                    url: place.webSearchUrl
+                })));
+            }
+
+            // Process time zone
+            if (parsedResponse.timeZone) {
+                results.push({
+                    searchResultId: getSearchResultId(),
+                    title: "Time Zone Information",
+                    content: parsedResponse.timeZone.primaryResponse || parsedResponse.timeZone.description
+                });
+            }
+
+            // Process translations
+            if (parsedResponse.translations && parsedResponse.translations.value) {
+                results.push(...parsedResponse.translations.value.map(translation => ({
+                    searchResultId: getSearchResultId(),
+                    title: "Translation",
+                    content: `Original (${translation.inLanguage}): ${translation.originalText}, Translated (${translation.translatedLanguageName}): ${translation.translatedText}`
+                })));
+            }
+
+            resolver.tool = JSON.stringify({ toolUsed: "Search" });
+            return JSON.stringify({ _type: "SearchResponse", value: results });
+        } catch (e) {
+            logger.error(`Error in Bing search: ${e}`);
+            throw e;
+        }
+    }
+};