@aj-archipelago/cortex 1.3.35 → 1.3.36

package/pathways/system/entity/tools/sys_tool_reasoning.js

@@ -0,0 +1,75 @@
+// sys_tool_reasoning.js
+// Entity tool that provides advanced reasoning and planning capabilities
+
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides advanced reasoning and planning capabilities. You excel at breaking down complex problems, creating detailed plans, and providing thorough analysis. Think carefully about the latest request and provide a detailed, well thought out, carefully reviewed response.\n{{renderTemplate AI_DATETIME}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 100000,
+    model: 'oai-o3',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: [{
+        type: "function",
+        icon: "🗺️",
+        function: {
+            name: "Plan",
+            description: "Use specifically to create a thorough, well thought out, step by step plan to accomplish a task. You should always use this tool when you're planning to do something complex or something that might require multiple steps.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    },
+    {
+        type: "function",
+        icon: "🧠",
+        function: {
+            name: "Reason",
+            description: "Employ for advancedreasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do" 
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    }],
+    
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        let result = await runAllPrompts({ ...args, stream: false });        
+        resolver.tool = JSON.stringify({ toolUsed: "reasoning" });          
+        return result;
+    }
+} 

package/pathways/system/entity/tools/sys_tool_remember.js

@@ -0,0 +1,59 @@
+// sys_tool_remember.js
+// Entity tool that looks for relevant information in the entity's memory
+import { callPathway } from '../../../../lib/pathwayTools.js';
+
+export default {
+    prompt:
+        [],
+    model: 'oai-gpt41-mini',
+
+    toolDefinition: [{
+        type: "function",
+        icon: "🧩",
+        function: {
+            name: "Remember",
+            description: "Use specifically to search your long term memory for information or details that may not be present in your short term memory. You should always use this tool before you tell the user you don't remember something. If the user asks you a question (like what's your favorite color) and you don't remember the answer, use this tool to search your long term memory for the answer before you tell the user you don't have one.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed description of what you want to see if you remember"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    },
+    {
+        type: "function",
+        icon: "🧩",
+        function: {
+            name: "LoadMemoryContext",
+            description: "This tool quickly preloads the memory context for this turn of the conversation.  It's typically automatically used by the system, but you can use it if you need to.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    }],
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        resolver.tool = JSON.stringify({ toolUsed: "memory" });
+        return await callPathway('sys_search_memory', { ...args, stream: false, section: 'memoryAll', updateContext: true });
+    }
+}

package/pathways/vision.js

@@ -13,7 +13,7 @@ export default {
         contextId: ``,
     },
     max_tokens: 1024,
-    model: 'oai-gpt4o',
+    model: 'oai-gpt41',
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,

package/server/modelExecutor.js

@@ -7,9 +7,6 @@ import AzureTranslatePlugin from './plugins/azureTranslatePlugin.js';
 import OpenAIWhisperPlugin from './plugins/openAiWhisperPlugin.js';
 import OpenAIChatExtensionPlugin from './plugins/openAiChatExtensionPlugin.js';
 import LocalModelPlugin from './plugins/localModelPlugin.js';
-import PalmChatPlugin from './plugins/palmChatPlugin.js';
-import PalmCompletionPlugin from './plugins/palmCompletionPlugin.js';
-import PalmCodeCompletionPlugin from './plugins/palmCodeCompletionPlugin.js';
 import CohereGeneratePlugin from './plugins/cohereGeneratePlugin.js';
 import CohereSummarizePlugin from './plugins/cohereSummarizePlugin.js';
 import AzureCognitivePlugin from './plugins/azureCognitivePlugin.js';
@@ -18,6 +15,7 @@ import OpenAIImagePlugin from './plugins/openAiImagePlugin.js';
 import OpenAIDallE3Plugin from './plugins/openAiDallE3Plugin.js';
 import OpenAIVisionPlugin from './plugins/openAiVisionPlugin.js';
 import OpenAIReasoningPlugin from './plugins/openAiReasoningPlugin.js';
+import OpenAIReasoningVisionPlugin from './plugins/openAiReasoningVisionPlugin.js';
 import GeminiChatPlugin from './plugins/geminiChatPlugin.js';
 import GeminiVisionPlugin from './plugins/geminiVisionPlugin.js';
 import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
@@ -70,15 +68,6 @@ class ModelExecutor {
             case 'LOCAL-CPP-MODEL':
                 plugin = new LocalModelPlugin(pathway, model);
                 break;
-            case 'PALM-CHAT':
-                plugin = new PalmChatPlugin(pathway, model);
-                break;
-            case 'PALM-COMPLETION':
-                plugin = new PalmCompletionPlugin(pathway, model);
-                break;
-            case 'PALM-CODE-COMPLETION':
-                plugin = new PalmCodeCompletionPlugin(pathway, model);
-                break;
             case 'COHERE-GENERATE':
                 plugin = new CohereGeneratePlugin(pathway, model);
                 break;
@@ -91,6 +80,9 @@ class ModelExecutor {
             case 'OPENAI-REASONING':
                 plugin = new OpenAIReasoningPlugin(pathway, model);
                 break;
+            case 'OPENAI-REASONING-VISION':
+                plugin = new OpenAIReasoningVisionPlugin(pathway, model);
+                break;
             case 'GEMINI-CHAT':
                 plugin = new GeminiChatPlugin(pathway, model);
                 break;

package/server/pathwayResolver.js

@@ -73,19 +73,18 @@ class PathwayResolver {
         this.pathwayPrompt = pathway.prompt;
     }
 
+    publishNestedRequestProgress(requestProgress) {
+        if (requestProgress.progress === 1 && this.rootRequestId) {
+            delete requestProgress.progress;
+        }
+        publishRequestProgress({...requestProgress, info: this.tool || ''});
+    }
+
     // This code handles async and streaming responses for either long-running
     // tasks or streaming model responses
     async asyncResolve(args) {
-        let streamErrorOccurred = false;
         let responseData = null;
 
-        const publishNestedRequestProgress = (requestProgress) => {
-            if (requestProgress.progress === 1 && this.rootRequestId) {
-                delete requestProgress.progress;
-            }
-            publishRequestProgress({...requestProgress, info: this.tool || ''});
-        }
-
         try {
             responseData = await this.executePathway(args);
         }
@@ -102,8 +101,45 @@ class PathwayResolver {
 
         // If the response is a stream, handle it as streaming response
         if (responseData && typeof responseData.on === 'function') {
+            await this.handleStream(responseData);
+        } else {
+            const { completedCount = 1, totalCount = 1 } = requestState[this.requestId];
+            requestState[this.requestId].data = responseData;
+            
+            // some models don't support progress updates
+            if (!modelTypesExcludedFromProgressUpdates.includes(this.model.type)) {
+                this.publishNestedRequestProgress({
+                        requestId: this.rootRequestId || this.requestId,
+                        progress: Math.min(completedCount, totalCount) / totalCount,
+                        // Clients expect these to be strings
+                        data: JSON.stringify(responseData || ''),
+                        info: this.tool || ''
+                });
+            }
+        }
+    }
+
+    mergeResults(mergeData) {
+        if (mergeData) {
+            this.previousResult = mergeData.previousResult ? mergeData.previousResult : this.previousResult;
+            this.warnings = [...this.warnings, ...(mergeData.warnings || [])];
+            this.errors = [...this.errors, ...(mergeData.errors || [])];
+            try {
+                const mergeDataTool = typeof mergeData.tool === 'string' ? JSON.parse(mergeData.tool) : mergeData.tool || {};
+                const thisTool = typeof this.tool === 'string' ? JSON.parse(this.tool) : this.tool || {};
+                this.tool = JSON.stringify({ ...thisTool, ...mergeDataTool });
+            } catch (error) {
+                logger.warn('Error merging pathway resolver tool objects: ' + error);
+            }
+        }
+    }
+
+    async handleStream(response) {
+        let streamErrorOccurred = false;
+
+        if (response && typeof response.on === 'function') {
             try {
-                const incomingMessage = responseData;
+                const incomingMessage = response;
                 let streamEnded = false;
 
                 const onParse = (event) => {
@@ -133,7 +169,7 @@ class PathwayResolver {
 
                     try {
                         if (!streamEnded && requestProgress.data) {
-                            publishNestedRequestProgress(requestProgress);
+                            this.publishNestedRequestProgress(requestProgress);
                             streamEnded = requestProgress.progress === 1;
                         }
                     } catch (error) {
@@ -173,35 +209,6 @@ class PathwayResolver {
             } else {
                 return;
             }
-        } else {
-            const { completedCount = 1, totalCount = 1 } = requestState[this.requestId];
-            requestState[this.requestId].data = responseData;
-            
-            // some models don't support progress updates
-            if (!modelTypesExcludedFromProgressUpdates.includes(this.model.type)) {
-                await publishNestedRequestProgress({
-                        requestId: this.rootRequestId || this.requestId,
-                        progress: Math.min(completedCount, totalCount) / totalCount,
-                        // Clients expect these to be strings
-                        data: JSON.stringify(responseData || ''),
-                        info: this.tool || ''
-                });
-            }
-        }
-    }
-
-    mergeResults(mergeData) {
-        if (mergeData) {
-            this.previousResult = mergeData.previousResult ? mergeData.previousResult : this.previousResult;
-            this.warnings = [...this.warnings, ...(mergeData.warnings || [])];
-            this.errors = [...this.errors, ...(mergeData.errors || [])];
-            try {
-                const mergeDataTool = typeof mergeData.tool === 'string' ? JSON.parse(mergeData.tool) : mergeData.tool || {};
-                const thisTool = typeof this.tool === 'string' ? JSON.parse(this.tool) : this.tool || {};
-                this.tool = JSON.stringify({ ...thisTool, ...mergeDataTool });
-            } catch (error) {
-                logger.warn('Error merging pathway resolver tool objects: ' + error);
-            }
         }
     }
 
@@ -212,7 +219,7 @@ class PathwayResolver {
                 requestState[this.requestId] = {}
             }
             this.rootRequestId = args.rootRequestId ?? null;
-            requestState[this.requestId] = { ...requestState[this.requestId], args, resolver: this.asyncResolve.bind(this) };
+            requestState[this.requestId] = { ...requestState[this.requestId], args, resolver: this.asyncResolve.bind(this), pathwayResolver: this };
             return this.requestId;
         }
         else {
@@ -297,6 +304,12 @@ class PathwayResolver {
                 break;
             }
 
+            // if data is a stream, handle it
+            if (data && typeof data.on === 'function') {
+                await this.handleStream(data);
+                return data;
+            }
+
             data = await this.responseParser.parse(data);
             if (data !== null) {
                 break;

package/server/plugins/azureBingPlugin.js

@@ -7,14 +7,42 @@ class AzureBingPlugin extends ModelPlugin {
         super(pathway, model);
     }
     
-    getRequestParameters(text) {
+    getRequestParameters(text, parameters = {}) {
+        const {
+            q, // Query string (takes precedence over text parameter)
+            responseFilter, // Comma-separated list of answer types to include/exclude
+            freshness, // 'day', 'week', 'month', or date range 'YYYY-MM-DD..YYYY-MM-DD'
+            answerCount, // Number of top answers to return
+            promote, // Comma-separated list of answer types to promote
+            count, // Number of webpages to return (default 10)
+            safeSearch = 'Moderate', // 'Off', 'Moderate', or 'Strict'
+        } = parameters;
+
         const requestParameters = {
-            data: [
-            ],
+            data: [],
             params: {
-                q: text,
+                q: q || text, // Use q if provided, otherwise fall back to text
             }
         };
+
+        // Add optional parameters if they exist
+        if (responseFilter) {
+            requestParameters.params.responseFilter = responseFilter;
+        }
+        if (freshness) {
+            requestParameters.params.freshness = freshness;
+        }
+        if (answerCount) {
+            requestParameters.params.answerCount = answerCount;
+        }
+        if (promote) {
+            requestParameters.params.promote = promote;
+        }
+        if (count) {
+            requestParameters.params.count = count;
+        }
+        requestParameters.params.safeSearch = safeSearch;
+
         return requestParameters;
     }
 
@@ -28,6 +56,16 @@ class AzureBingPlugin extends ModelPlugin {
         cortexRequest.params = requestParameters.params;
         cortexRequest.method = 'GET';
 
+        // Step 1: Strip any existing endpoint after version number
+        cortexRequest.url = cortexRequest.url.replace(/\/v(\d+\.\d+)\/.*$/, '/v$1');
+        
+        // Step 2: Add appropriate endpoint based on searchType
+        if (parameters.searchType === 'news') {
+            cortexRequest.url += '/news/search';
+        } else {
+            cortexRequest.url += '/search';
+        }
+
         return this.executeRequest(cortexRequest);
     }
     

package/server/plugins/azureCognitivePlugin.js

@@ -34,7 +34,7 @@ class AzureCognitivePlugin extends ModelPlugin {
     async getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId, cortexRequest) {
         const combinedParameters = { ...this.promptParameters, ...parameters };
         const { modelPromptText } = this.getCompiledPrompt(text, combinedParameters, prompt);
-        const { inputVector, calculateInputVector, privateData, filter, docId, title, chunkNo, chatId } = combinedParameters;
+        const { inputVector, calculateInputVector, privateData, filter, docId, title, chunkNo, chatId, semanticConfiguration } = combinedParameters;
         const data = {};
 
         if (mode == 'delete') {
@@ -105,18 +105,45 @@ class AzureCognitivePlugin extends ModelPlugin {
         }
 
         //default mode, 'search'
-        if (inputVector) {
-            data.vectors = [
-                {
-                    "value": typeof inputVector === 'string' ? JSON.parse(inputVector) : inputVector,
-                    "fields": "contentVector",
-                    "k": 20
-                }
-            ];
+        data.search = modelPromptText;
+        data.top = parameters.top || 50;
+        data.skip = 0;
+        data.count = true;
+
+        // If semanticConfiguration is provided, switch to semantic mode
+        if (semanticConfiguration) {
+            data.queryType = "semantic";
+            data.semanticConfiguration = semanticConfiguration; // Use provided value directly
+            data.captions = "extractive";
+            data.answers = "extractive|count-3";
+            data.queryLanguage = "en-us";
+            // Omit top-level queryRewrites as it caused issues before
+
+            if (inputVector) {
+                // Use vectorQueries for semantic hybrid search
+                data.vectorQueries = [
+                    {
+                        "kind": "text",
+                        "text": modelPromptText, // Use the search text for the vector query
+                        "fields": "contentVector", // Ensure this field name is correct for your index
+                        "k": parameters.k || 20, // Use parameter k or default
+                        "queryRewrites": "generative" // Add queryRewrites inside vector query
+                    }
+                ];
+                delete data.vectors; // Remove the standard vector field
+            }
         } else {
-            data.search = modelPromptText;
-            data.top = parameters.top || 50;
-            data.skip = 0;
+            // Standard non-semantic search
+            if (inputVector) {
+                data.vectors = [
+                    {
+                        "value": typeof inputVector === 'string' ? JSON.parse(inputVector) : inputVector,
+                        "fields": "contentVector",
+                        "k": parameters.k || 20
+                    }
+                ];
+            }
+            // Handle titleOnly only for non-semantic search
             if (parameters.titleOnly) {
                 switch(indexName){
                     case 'indexcortex':
@@ -130,6 +157,7 @@ class AzureCognitivePlugin extends ModelPlugin {
             }
         }
 
+        // Apply filters (common to both semantic and non-semantic)
         filter && (data.filter = filter);
         if (indexName == 'indexcortex') { //if private, filter by owner via contextId //privateData && 
             data.filter && (data.filter = data.filter + ' and ');

package/server/plugins/claude3VertexPlugin.js

@@ -115,13 +115,32 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
     const { content } = data;
 
-    // if the response is an array, return the text property of the first item
-    // if the type property is 'text'
-    if (content && Array.isArray(content) && content[0].type === "text") {
-      return content[0].text;
-    } else {
-      return data;
+    // Handle tool use responses from Claude
+    if (content && Array.isArray(content)) {
+      const toolUses = content.filter(item => item.type === "tool_use");
+      if (toolUses.length > 0) {
+        return {
+          role: "assistant",
+          content: "",
+          tool_calls: toolUses.map(toolUse => ({
+            id: toolUse.id,
+            type: "function",
+            function: {
+              name: toolUse.name,
+              arguments: JSON.stringify(toolUse.input)
+            }
+          }))
+        };
+      }
+
+      // Handle regular text responses
+      const textContent = content.find(item => item.type === "text");
+      if (textContent) {
+        return textContent.text;
+      }
     }
+
+    return data;
   }
 
   // This code converts messages to the format required by the Claude Vertex API
@@ -271,17 +290,44 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
       });
     }
 
+    if (parameters.tool_choice) {
+      // Convert OpenAI tool_choice format to Claude format
+      if (typeof parameters.tool_choice === 'string') {
+        // Handle string values: auto, required, none
+        if (parameters.tool_choice === 'required') {
+          requestParameters.tool_choice = { type: 'any' }; // OpenAI's 'required' maps to Claude's 'any'
+        } else if (parameters.tool_choice === 'auto') {
+          requestParameters.tool_choice = { type: 'auto' };
+        } else if (parameters.tool_choice === 'none') {
+          requestParameters.tool_choice = { type: 'none' };
+        }
+      } else if (parameters.tool_choice.type === "function") {
+        // Handle function-specific tool choice
+        requestParameters.tool_choice = {
+          type: "tool",
+          name: parameters.tool_choice.function.name
+        };
+      }
+    }
+
     // If there are function calls in messages, generate tools block
     if (modifiedMessages?.some(msg => 
       Array.isArray(msg.content) && msg.content.some(item => item.type === 'tool_use')
     )) {
       const toolsMap = new Map();
       
-      // Collect all unique tool uses from messages
+      // First add any existing tools from parameters to the map
+      if (requestParameters.tools) {
+        requestParameters.tools.forEach(tool => {
+          toolsMap.set(tool.name, tool);
+        });
+      }
+      
+      // Collect all unique tool uses from messages, only adding if not already present
       modifiedMessages.forEach(msg => {
         if (Array.isArray(msg.content)) {
           msg.content.forEach(item => {
-            if (item.type === 'tool_use') {
+            if (item.type === 'tool_use' && !toolsMap.has(item.name)) {
               toolsMap.set(item.name, {
                 name: item.name,
                 description: `Tool for ${item.name}`,
@@ -302,11 +348,8 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
         }
       });
       
-      if (requestParameters.tools) {
-        requestParameters.tools.push(...Array.from(toolsMap.values()));
-      } else {
-        requestParameters.tools = Array.from(toolsMap.values());
-      }
+      // Update the tools array with the combined unique tools
+      requestParameters.tools = Array.from(toolsMap.values());
     }
 
     requestParameters.max_tokens = this.getModelMaxReturnTokens();
@@ -361,11 +404,17 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     if (stream) {
       logger.info(`[response received as an SSE stream]`);
     } else {
-      const responseText = this.parseResponse(responseData);
-      const { length, units } = this.getLength(responseText);
-      logger.info(`[response received containing ${length} ${units}]`);
-      logger.verbose(`${responseText}`);
-    }
+      const parsedResponse = this.parseResponse(responseData);
+      
+      if (typeof parsedResponse === 'string') {
+          const { length, units } = this.getLength(parsedResponse);
+          logger.info(`[response received containing ${length} ${units}]`);
+          logger.verbose(`${this.shortenContent(parsedResponse)}`);
+      } else {
+          logger.info(`[response received containing object]`);
+          logger.verbose(`${JSON.stringify(parsedResponse)}`);
+      }
+  }
 
     prompt &&
       prompt.debugInfo &&

package/server/plugins/modelPlugin.js

@@ -565,11 +565,12 @@ class ModelPlugin {
             return parsedData;
         } catch (error) {
             // Log the error and continue
-            logger.error(`Error in executeRequest for ${this.pathwayName}: ${error.message || error}`);
+            const errorMessage = `${error?.response?.data?.message || error?.response?.data?.error?.message || error?.message || error}`;
+            logger.error(`Error in executeRequest for ${this.pathwayName}: ${errorMessage}`);
             if (error.data) {
                 logger.error(`Additional error data: ${JSON.stringify(error.data)}`);
             }
-            throw new Error(`Execution failed for ${this.pathwayName}: ${error.message || error}`);
+            throw new Error(`Execution failed for ${this.pathwayName}: ${errorMessage}`);
         }
     }
 

package/server/plugins/openAiReasoningPlugin.js

@@ -11,7 +11,7 @@ class OpenAIReasoningPlugin extends OpenAIChatPlugin {
                     role: message.role,
                     content: this.parseContent(message.content)
                 });
-            }else if(message.role === 'system') {
+            } else if (message.role === 'system') {
                 // System messages to developer: https://platform.openai.com/docs/guides/text-generation#messages-and-roles
                 newMessages.push({
                     role: "developer",
@@ -20,8 +20,8 @@ class OpenAIReasoningPlugin extends OpenAIChatPlugin {
             }
         }
 
-        messages.length = 0;
-        messages.push(...newMessages);
+        // Replace the contents of the original array with the new messages
+        messages.splice(0, messages.length, ...newMessages);
     }
 
     parseContent(content) {