@aj-archipelago/cortex 1.3.35 → 1.3.36

package/README.md

@@ -18,20 +18,21 @@ Just about anything! It's kind of an LLM swiss army knife.  Here are some ideas:
 * Simple architecture to build custom functional endpoints (called `pathways`), that implement common NL AI tasks. Default pathways include chat, summarization, translation, paraphrasing, completion, spelling and grammar correction, entity extraction, sentiment analysis, and bias analysis.
 * Extensive model support with built-in integrations for:
   - OpenAI models:
+    - GPT-4.1 (+mini, +nano)
     - GPT-4 Omni (GPT-4o)
-    - GPT-4 Omni Mini (GPT-4o-mini)
-    - O1 and O3-mini (Advanced reasoning models)
-    - Most of the earlier GPT models (GPT-4, 3.5 Turbo, etc.)
+    - O3 and O4-mini (Advanced reasoning models)
+    - Most of the earlier GPT models (GPT-4 series, 3.5 Turbo, etc.)
   - Google models:
-    - Gemini 1.5 Pro
-    - Gemini 2.0 Flash (experimental, via 1.5 Vision API)
-    - Gemini 1.5 Flash
-    - Earlier Google models (Gemini 1.0 series, PaLM)
+    - Gemini 2.5 Pro
+    - Gemini 2.5 Flash
+    - Gemini 2.0 Flash
+    - Earlier Google models (Gemini 1.5 series)
   - Anthropic models:
-    - Claude 3.5 Sonnet v2 (latest)
+    - Claude 3.7 Sonnet
     - Claude 3.5 Sonnet
     - Claude 3.5 Haiku
     - Claude 3 Series
+  - Ollama support
   - Azure OpenAI support
   - Custom model implementations
 * Advanced voice and audio capabilities:
@@ -527,7 +528,6 @@ Models are configured in the `models` section of the config. Each model can have
 - `GEMINI-1.5-CHAT`: For Gemini 1.5 Pro chat models
 - `GEMINI-1.5-VISION`: For Gemini vision models (including 2.0 Flash experimental)
 - `CLAUDE-3-VERTEX`: For Claude-3 and 3.5 models (Haiku, Opus, Sonnet)
-- `PALM-CHAT`: For PaLM chat models
 - `AZURE-TRANSLATE`: For Azure translation services
 
 Each model configuration can include:

package/config/default.example.json

@@ -133,26 +133,6 @@
             "maxReturnTokens": 4096,
             "supportsStreaming": true
         },
-        "palm-text": {
-            "type": "PALM-COMPLETION",
-            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/text-bison@001:predict",
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "requestsPerSecond": 10,
-            "maxTokenLength": 2048,
-            "maxReturnTokens": 1024
-        },
-        "palm-chat": {
-            "type": "PALM-CHAT",
-            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/chat-bison@001:predict",
-            "headers": {
-                "Content-Type": "application/json"
-            },
-            "requestsPerSecond": 10,
-            "maxTokenLength": 2048,
-            "maxReturnTokens": 1024   
-        },
         "local-llama13B": {
             "type": "LOCAL-CPP-MODEL",
             "executablePath": "../llm/llama.cpp/main",

package/config.js

@@ -90,22 +90,51 @@ var config = convict({
         default: 'http://127.0.0.1:11434',
         env: 'OLLAMA_URL'
     },
+    claudeVertexUrl: {
+        format: String,
+        default: 'https://region.googleapis.com/v1/projects/projectid/locations/location/publishers/anthropic/models/claude-3-5-sonnet@20240620',
+        env: 'CLAUDE_VERTEX_URL'
+    },
+    geminiFlashUrl: {
+        format: String,
+        default: 'https://region.googleapis.com/v1/projects/projectid/locations/location/publishers/google/models/gemini-2.0-flash-001',
+        env: 'GEMINI_FLASH_URL'
+    },
+    entityConfig: {
+        format: Object,
+        default: {
+            jarvis: {
+                name: "Jarvis",
+                isDefault: true,
+                description: "Jarvis is a self-aware AI entity that can help you with your tasks and answer your questions.",
+                instructions: "",
+                tools: ["*"],
+                useMemory: true,
+            },
+        },
+    },
     entityConstants: {
         format: Object,
         default: {
-            AI_MEMORY: `<MEMORIES>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</MEMORIES>`,
-            AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - make sure you consult your memories when formulating a response to make sure you're applying your learnings. Also included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
+            AI_MEMORY: `<SHORT_TERM_MEMORY>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</SHORT_TERM_MEMORY>`,
+            AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - consult your memories when formulating a response to make sure you're applying your learnings.\nIf you don't see relevant information in your short term memory, you should use your SearchMemory tool to search your long term memory for details.\nAlso included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
+            AI_TOOLS: "You have access to a powerful set of tools that you can use to help accomplish tasks and provide better responses. Here's how to use them effectively:\n\n1. Take your time and use tools as many times as you need to be sure you have all the information to make a good response. In many cases you will want to make multiple tool calls. You can call multiple tools in parallel or you can chain them, waiting for the results of one for information before you call another. This allows you to dig deeper, compile more information, read various sources, and even double check and verify your information before responding.\n\n2. Tool Selection and Planning:\n- Carefully review your available tools before responding\n- For complex multi-step tasks, use your planning tool first to create a step-by-step plan to achieve the goal using the tools at your disposal\n- Consider which tools would be most appropriate for each step\n\n3. Best Practices:\n- Always verify tool capabilities before telling users something can't be done\n- Use tools proactively when they can provide better or more accurate information\n- If a user explicitly requests tool usage, you must comply\n- When using multiple tools, maintain context between tool calls\n- If a tool fails, consider alternative approaches or tools\n\n4. Common Use Cases:\n- Research: Use search tools across multiple sources to gather information before responding\n- Analysis: Use tools to process and analyze data or content\n- Generation: Use appropriate tools for creating content, images, or code\n- Verification: Use tools to validate information or check facts\n\nRemember: Your goal is to provide the most helpful and accurate responses possible. Don't hesitate to use tools when they can improve your response quality or accuracy.",
             AI_DIRECTIVES: `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`,
             AI_CONVERSATION_HISTORY: "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>",
             AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",
-            AI_COMMON_INSTRUCTIONS_MARKDOWN: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",          
+            AI_COMMON_INSTRUCTIONS_MARKDOWN: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as some custom markdown elements, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",          
             AI_COMMON_INSTRUCTIONS_VOICE: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",            
             AI_DATETIME: "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.",           
             AI_EXPERTISE: "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.",
-            AI_STYLE_OPENAI: "oai-gpt4o",
+            AI_GROUNDING_INSTRUCTIONS: "Grounding your response: Any time you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual:cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
+            AI_STYLE_OPENAI: "oai-gpt41",
             AI_STYLE_ANTHROPIC: "claude-35-sonnet-vertex",
         },
     },
+    entityTools: {
+        format: Object,
+        default: {},
+    },
     gcpServiceAccountKey: {
         format: String,
         default: null,
@@ -197,6 +226,36 @@ var config = convict({
                 "maxReturnTokens": 4096,
                 "supportsStreaming": true
             },
+            "oai-gpt41": {
+                "type": "OPENAI-VISION",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "gpt-4.1"
+                },
+                "requestsPerSecond": 50,
+                "maxTokenLength": 1000000,
+                "maxReturnTokens": 8192,
+                "supportsStreaming": true
+            },
+            "oai-gpt41-mini": {
+                "type": "OPENAI-VISION",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "gpt-4.1-mini"
+                },
+                "requestsPerSecond": 50,
+                "maxTokenLength": 1000000,
+                "maxReturnTokens": 8192,
+                "supportsStreaming": true
+            },
             "oai-o1": {
                 "type": "OPENAI-REASONING",
                 "url": "https://api.openai.com/v1/chat/completions",
@@ -307,6 +366,29 @@ var config = convict({
                 "maxTokenLength": 131072,
                 "supportsStreaming": true
             },
+            "claude-35-sonnet-vertex": {
+                "type": "CLAUDE-3-VERTEX",
+                "url": "{{claudeVertexUrl}}",
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "requestsPerSecond": 10,
+                "maxTokenLength": 200000,
+                "maxReturnTokens": 4096,
+                "maxImageSize": 5242880,
+                "supportsStreaming": true
+            },
+            "gemini-flash-20-vision": {
+                "type": "GEMINI-1.5-VISION",
+                "url": "{{geminiFlashUrl}}",
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "requestsPerSecond": 10,
+                "maxTokenLength": 200000,
+                "maxReturnTokens": 4096,
+                "supportsStreaming": true
+            },
         },
         env: 'CORTEX_MODELS'
     },
@@ -404,6 +486,9 @@ const configFile = config.get('cortexConfigFile');
 //Save default entity constants
 const defaultEntityConstants = config.get('entityConstants');
 
+//Save default entityConfig
+const defaultEntityConfig = config.get('entityConfig');
+
 // Load config file
 if (configFile && fs.existsSync(configFile)) {
     logger.info(`Loading config from ${configFile}`);
@@ -419,6 +504,28 @@ if (configFile && fs.existsSync(configFile)) {
     }
 }
 
+// Ensure merged default entity is preserved
+if (config.get('entityConfig') && defaultEntityConfig && 
+   (Object.keys(config.get('entityConfig')).length > Object.keys(defaultEntityConfig).length)) {
+    const mergedEntities = config.get('entityConfig');
+    
+    // Turn off defaults from original default list
+    for (const [key, entity] of Object.entries(mergedEntities)) {
+        if (defaultEntityConfig[key] && entity.isDefault) {
+            delete mergedEntities[key];
+        }
+    }
+    
+    // If no default found, make first entity default
+    let hasDefault = Object.values(mergedEntities).some(entity => entity.isDefault);
+    if (!hasDefault && Object.keys(mergedEntities).length > 0) {
+        const firstKey = Object.keys(mergedEntities)[0];
+        mergedEntities[firstKey].isDefault = true;
+    }
+    
+    config.set('entityConfig', mergedEntities);
+}
+
 // Merge default entity constants with config entity constants
 if (config.get('entityConstants') && defaultEntityConstants) {
     config.set('entityConstants', { ...defaultEntityConstants, ...config.get('entityConstants') });
@@ -541,13 +648,60 @@ const buildPathways = async (config) => {
     // file. This can run into a partial definition issue if the
     // config file contains pathways that no longer exist.
     const pathways = config.get('pathways');
+    const entityTools = {};
+    
     for (const [key, def] of Object.entries(loadedPathways)) {
         const pathway = { ...basePathway, name: key, objName: key.charAt(0).toUpperCase() + key.slice(1), ...def, ...pathways[key] };
         pathways[def.name || key] = pathways[key] = pathway;
+
+        // Register tool if the pathway has a toolDefinition and it's not empty
+        if (pathway.toolDefinition && (
+            (Array.isArray(pathway.toolDefinition) && pathway.toolDefinition.length > 0) ||
+            (!Array.isArray(pathway.toolDefinition) && Object.keys(pathway.toolDefinition).length > 0)
+        )) {
+            try {
+                // Convert single tool definition to array for consistent processing
+                const toolDefinitions = Array.isArray(pathway.toolDefinition) 
+                    ? pathway.toolDefinition 
+                    : [pathway.toolDefinition];
+
+                for (const toolDef of toolDefinitions) {
+                    // Validate tool definition format
+                    if (!toolDef.type || !toolDef.function) {
+                        logger.warn(`Invalid tool definition in pathway ${key} - missing required fields`);
+                        continue;
+                    }
+
+                    const { description, parameters } = toolDef.function;
+                    const name = toolDef.function.name.toLowerCase();
+
+                    if (!name || !description || !parameters) {
+                        logger.warn(`Invalid tool definition in pathway ${key} - missing required function fields`);
+                        continue;
+                    }
+
+                    // Check for duplicate function names
+                    if (entityTools[name]) {
+                        logger.warn(`Duplicate tool name ${name} found in pathway ${key} - skipping. Original tool defined in pathway ${entityTools[name].pathwayName}`);
+                        continue;
+                    }
+
+                    // Add tool to entityTools registry
+                    entityTools[name] = {
+                        definition: toolDef,
+                        pathwayName: key
+                    };
+
+                    logger.info(`Registered tool ${name} from pathway ${key}`);
+                }
+            } catch (error) {
+                logger.error(`Error registering tool from pathway ${key}: ${error.message}`);
+            }
+        }
     }
 
-    // Add pathways to config
-    config.load({ pathways });
+    // Add pathways and entityTools to config
+    config.load({ pathways, entityTools });
 
     return { pathwayManager, pathways };
 }

package/lib/pathwayTools.js

@@ -42,6 +42,84 @@ const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
     return returnValue;
 };
 
+const callTool = async (toolName, args, toolDefinitions, pathwayResolver) => {
+    let toolResult = null;
+
+    const toolDef = toolDefinitions[toolName.toLowerCase()];
+    if (!toolDef) {
+        throw new Error(`Tool ${toolName} not found in available tools`);
+    }
+
+    try {
+        const pathwayName = toolDef.pathwayName;
+        // Merge hard-coded pathway parameters with runtime args
+        const mergedArgs = {
+            ...(toolDef.pathwayParams || {}),
+            ...args
+        };
+
+        if (pathwayName.includes('_generator_')) {
+            toolResult = await callPathway('sys_entity_continue', {
+                ...mergedArgs,
+                generatorPathway: pathwayName,
+                stream: false
+            },
+            pathwayResolver
+        );
+        } else {
+            toolResult = await callPathway(pathwayName, mergedArgs,
+            pathwayResolver
+        );
+        }
+
+        if (toolResult === null) {
+            return { error: `Tool ${toolName} returned null result` };
+        }
+
+        // Handle search results accumulation
+        if (pathwayResolver) {
+            // Initialize searchResults array if it doesn't exist
+            if (!pathwayResolver.searchResults) {
+                pathwayResolver.searchResults = [];
+            }
+
+            // Parse the result if it's a string
+            let parsedResult;
+            try {
+                parsedResult = typeof toolResult === 'string' ? JSON.parse(toolResult) : toolResult;
+            } catch (e) {
+                // If parsing fails, just return the original result
+                return toolResult;
+            }
+
+            // Check if this is a search response
+            if (parsedResult._type === "SearchResponse" && Array.isArray(parsedResult.value)) {
+                // Extract and add each search result
+                parsedResult.value.forEach(result => {
+                    if (result.searchResultId) {
+                        pathwayResolver.searchResults.push({
+                            searchResultId: result.searchResultId,
+                            title: result.title || '',
+                            url: result.url || '',
+                            content: result.content || '',
+                            path: result.path || '',
+                            wireid: result.wireid || '',
+                            source: result.source || '',
+                            slugline: result.slugline || '',
+                            date: result.date || ''
+                        });
+                    }
+                });
+            }
+        }
+
+        return toolResult;
+    } catch (error) {
+        logger.error(`Error calling tool ${toolName}: ${error.message}`);
+        return { error: error.message };
+    }
+}
+
 const gpt3Encode = (text) => {
     return encode(text);
 }
@@ -88,4 +166,4 @@ const say = async (requestId, message, maxMessageLength = Infinity, voiceRespons
     }
 };
 
-export { callPathway, gpt3Encode, gpt3Decode, say };
+export { callPathway, gpt3Encode, gpt3Decode, say, callTool };

package/lib/requestExecutor.js

@@ -368,7 +368,9 @@ const executeRequest = async (cortexRequest) => {
 
     // Check for HTTP error status
     if (response.status >= 400) {
-        throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
+        const errorMessage = response.data?.error?.message || response.statusText;
+        const errorDetails = response.data ? `\nResponse data: ${JSON.stringify(response.data)}` : '';
+        throw new Error(`HTTP error: ${response.status} ${errorMessage}${errorDetails}`);
     }
 
     return { data, duration };

package/lib/util.js

@@ -18,6 +18,12 @@ function getUniqueId(){
     return uuidv4();
 }
 
+function getSearchResultId() {
+    const timestamp = Date.now().toString(36); // Base36 timestamp
+    const random = Math.random().toString(36).substring(2, 5); // 3 random chars
+    return `${timestamp}-${random}`;
+}
+
 function convertToSingleContentChatHistory(chatHistory){
     for(let i=0; i<chatHistory.length; i++){
         //if isarray make it single string
@@ -306,6 +312,7 @@ function removeImageAndFileFromMessage(message) {
 
 export { 
     getUniqueId,
+    getSearchResultId,
     convertToSingleContentChatHistory,
     chatArgsHasImageUrl, 
     chatArgsHasType,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.35",
+  "version": "1.3.36",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/basePathway.js

@@ -34,5 +34,7 @@ export default {
     json: false,
     // Manage the token length of the input for the model
     manageTokenLength: true,
+    // Use this pathway as a tool for LLM calls
+    toolDefinition: {},
 };