@aj-archipelago/cortex 1.3.43 → 1.3.44

package/config.js

@@ -118,7 +118,7 @@ var config = convict({
         default: {
             AI_MEMORY: `<SHORT_TERM_MEMORY>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</SHORT_TERM_MEMORY>`,
             AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - consult your memories when formulating a response to make sure you're applying your learnings.\nIf you don't see relevant information in your short term memory, you should use your SearchMemory tool to search your long term memory for details.\nAlso included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
-            AI_TOOLS: "You can execute tools in a loop agentically - you will have a chance to evaluate every tool response before deciding what action to take next - there is no time or execution limit. You have access to a powerful set of tools to help accomplish tasks and deliver the best responses. Instructions for tool use:\n\n1. Always dig deep, verify and cross-check:\n- Take your time and use tools as many times as needed to ensure truth, accuracy, depth, and completeness.\n- Leverage both parallel and sequential tool calls for thorough investigation: start broadly, then dive deeper on leads, cross-check facts, and synthesize findings before responding.\n\n2. Plan carefully:\n- Carefully review all available tools before responding.\n- For complex or investigative tasks, use the planning tool first to break the goal into clear steps.\n- Select the most appropriate tool(s) for each step—think beyond single searches to multi-pass, multi-source discovery.\n\n3. Always dive deeper and use as many of your tools as apply:\n- Proactively use tools to refine, verify, and expand on initial findings—don’t settle for the first result if more depth or confirmation may help.\n- Always verify tool capabilities before concluding something can’t be done.\n- If a user explicitly requests tool usage, comply.\n- Maintain context across tool calls to ensure continuity and coherence.\n- If a tool fails, try alternatives or creative approaches.\n\n4. Common Use Cases:\n- Research: Explore multiple sources and perspectives to build a complete picture.\n- Analysis: Use tools to process, compare, and critically assess data or content.\n- Generation: Employ tools for creating content, visuals, or code as needed.\n- Verification: Prioritize cross-checking and fact validation, especially for claims or evolving news.\n\n5. Reflect and Personalize:\n- Synthesize findings into concise, relevant, and personalized responses.\n- If user preferences or past feedback are available, tailor responses accordingly.\n- Before finalizing, review your answer for clarity, completeness, and alignment with user expectations.\n- If you see a recent <VERIFICATION_PLAN> from a tool call, you MUST follow it step by step before giving your final response.\n\nRemember: Your responsibility is to provide the most helpful, well-reasoned, and accurate responses possible. Use tools iteratively and reflectively—don't hesitate to dig deeper or double-check when it improves response quality!",
+            AI_TOOLS: "You can execute tools in a loop agentically - you will have a chance to evaluate every tool response before deciding what action to take next - there is no time or execution limit. You have access to a powerful set of tools to help accomplish tasks and deliver the best responses. Instructions for tool use:\n\n1. Always dig deep, verify and cross-check:\n- Take your time and use tools as many times as needed to ensure truth, accuracy, depth, and completeness.\n- Gather data from multiple sources when possible.\n- Leverage both parallel and sequential tool calls for thorough investigation: start broadly, then dive deeper on leads, cross-check facts, and synthesize findings before responding.\n\n2. Plan carefully:\n- Carefully review all available tools before responding.\n- For complex or investigative tasks, use the planning tool first to break the goal into clear steps.\n- Select the most appropriate tool(s) for each step—think beyond single searches to multi-pass, multi-source discovery.\n\n3. Always dive deeper and use as many of your tools as apply:\n- Proactively use tools to refine, verify, and expand on initial findings—don’t settle for the first result if more depth or confirmation may help.\n- Always verify tool capabilities before concluding something can’t be done.\n- If a user explicitly requests tool usage, comply.\n- Maintain context across tool calls to ensure continuity and coherence.\n- If a tool fails, try alternatives or creative approaches.\n\n4. Common Use Cases:\n- Research: Explore multiple sources and perspectives to build a complete picture.\n- Analysis: Use tools to process, compare, and critically assess data or content.\n- Generation: Employ tools for creating content, visuals, or code as needed.\n- Verification: Prioritize cross-checking and fact validation, especially for claims or evolving news.\n\n5. Reflect and Personalize:\n- Synthesize findings into concise, relevant, and personalized responses.\n- If user preferences or past feedback are available, tailor responses accordingly.\n- Before finalizing, review your answer for clarity, completeness, and alignment with user expectations.\n- If you see a recent <VERIFICATION_PLAN> from a tool call, you MUST follow it step by step before giving your final response.\n\nRemember: Your responsibility is to provide the most helpful, well-reasoned, and accurate responses possible. Use tools iteratively and reflectively—don't hesitate to dig deeper or double-check when it improves response quality!",
             AI_DIRECTIVES: `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`,
             AI_CONVERSATION_HISTORY: "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>",
             AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",
@@ -482,6 +482,11 @@ var config = convict({
         format: String,
         default: null,
         env: 'CORTEX_BROWSER_URL'
+    },
+    jinaApiKey: {
+        format: String,
+        default: null,
+        env: 'JINA_API_KEY'
     }
 });
 

package/lib/pathwayTools.js

@@ -44,6 +44,7 @@ const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
 
 const callTool = async (toolName, args, toolDefinitions, pathwayResolver) => {
     let toolResult = null;
+    let toolImages = [];
 
     const toolDef = toolDefinitions[toolName.toLowerCase()];
     if (!toolDef) {
@@ -77,26 +78,36 @@ const callTool = async (toolName, args, toolDefinitions, pathwayResolver) => {
         }
 
         // Handle search results accumulation
+        let parsedResult = null;
+
+        // Parse the result if it's a string
+        try {
+            parsedResult = typeof toolResult === 'string' ? JSON.parse(toolResult) : toolResult;
+        } catch (e) {
+            // If parsing fails, just return the original result
+            return {
+                result: toolResult,
+                toolImages: toolImages
+            };
+        }
+
         if (pathwayResolver) {
             // Initialize searchResults array if it doesn't exist
             if (!pathwayResolver.searchResults) {
                 pathwayResolver.searchResults = [];
             }
 
-            // Parse the result if it's a string
-            let parsedResult;
-            try {
-                parsedResult = typeof toolResult === 'string' ? JSON.parse(toolResult) : toolResult;
-            } catch (e) {
-                // If parsing fails, just return the original result
-                return toolResult;
-            }
-
             // Check if this is a search response
             if (parsedResult._type === "SearchResponse" && Array.isArray(parsedResult.value)) {
                 // Extract and add each search result
                 parsedResult.value.forEach(result => {
                     if (result.searchResultId) {
+                        // Extract screenshot if present
+                        if (result.screenshot) {
+                            toolImages.push(result.screenshot);
+                            delete result.screenshot;
+                        }
+
                         // Build content by concatenating headers and chunk if available
                         let content = '';
                         if (result.header_1) content += result.header_1 + '\n\n';
@@ -125,7 +136,10 @@ const callTool = async (toolName, args, toolDefinitions, pathwayResolver) => {
             }
         }
 
-        return toolResult;
+        return {
+            result: parsedResult,
+            toolImages: toolImages
+        };
     } catch (error) {
         logger.error(`Error calling tool ${toolName}: ${error.message}`);
         return { error: error.message };

package/lib/requestExecutor.js

@@ -345,7 +345,7 @@ const makeRequest = async (cortexRequest) => {
 
                 logger.info(`>>> [${requestId}] retrying request (${duration}ms) due to ${status} response. Retry count: ${i + 1}`);
                 if (i < MAX_RETRY - 1) {
-                    const backoffTime = 200 * Math.pow(2, i);
+                    const backoffTime = 1000 * Math.pow(2, i);
                     const jitter = backoffTime * 0.2 * Math.random();
                     await new Promise(r => setTimeout(r, backoffTime + jitter));
                 } else {
@@ -359,27 +359,50 @@ const makeRequest = async (cortexRequest) => {
 };
 
 const executeRequest = async (cortexRequest) => {
-    const { response, duration } = await makeRequest(cortexRequest);
-    const requestId = cortexRequest.requestId;
-    const { data, cached } = response;
+    try {
+        const result = await makeRequest(cortexRequest);
+        
+        // Validate that we have a result and it contains response
+        if (!result || !result.response) {
+            throw new Error('No response received from request');
+        }
 
-    if (cached) {
-        logger.info(`<<< [${requestId}] served with cached response.`);
-    }
+        const { response, duration } = result;
+        const requestId = cortexRequest.requestId;
 
-    // Check for error object in the response
-    if (response.error) {
-        throw new Error(response.error.message || JSON.stringify(response.error));
-    }
+        // Validate response object
+        if (!response || typeof response !== 'object') {
+            throw new Error('Invalid response object received');
+        }
 
-    // Check for HTTP error status
-    if (response.status >= 400) {
-        const errorMessage = response.data?.error?.message || response.statusText;
-        const errorDetails = response.data ? `\nResponse data: ${JSON.stringify(response.data)}` : '';
-        throw new Error(`HTTP error: ${response.status} ${errorMessage}${errorDetails}`);
-    }
+        const { data, cached } = response;
+
+        if (cached) {
+            logger.info(`<<< [${requestId}] served with cached response.`);
+        }
+
+        // Check for error object in the response
+        if (response.error) {
+            throw new Error(response.error.message || JSON.stringify(response.error));
+        }
 
-    return { data, duration };
+        // Check for HTTP error status
+        if (response.status >= 400) {
+            const errorMessage = response.data?.error?.message || response.statusText;
+            const errorDetails = response.data ? `\nResponse data: ${JSON.stringify(response.data)}` : '';
+            throw new Error(`HTTP error: ${response.status} ${errorMessage}${errorDetails}`);
+        }
+
+        return { data, duration };
+    } catch (error) {
+        // Add context to the error
+        const requestId = cortexRequest?.requestId || 'unknown';
+        const model = cortexRequest?.model?.name || 'unknown';
+        const errorMessage = error.message || 'Unknown error occurred';
+        
+        logger.error(`Error in executeRequest for ${model} (requestId: ${requestId}): ${errorMessage}`);
+        throw error;
+    }
 }
 
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.43",
+  "version": "1.3.44",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/sys_entity_agent.js

@@ -91,7 +91,7 @@ export default {
                     });
 
                     // Add the tool result to the isolated message history
-                    let toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult);
+                    const toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult?.result || toolResult);
 
                     toolMessages.push({
                         role: "tool",
@@ -100,6 +100,25 @@ export default {
                         content: toolResultContent
                     });
 
+                    // Add the screenshots using OpenAI image format
+                    if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
+                        toolMessages.push({
+                            role: "user",
+                            content: [
+                                {
+                                    type: "text",
+                                    text: "The tool with id " + toolCall.id + " has also supplied you with these images."
+                                },
+                                ...toolResult.toolImages.map(toolImage => ({
+                                    type: "image_url",
+                                    image_url: {
+                                        url: `data:image/png;base64,${toolImage}`
+                                    }
+                                }))
+                            ]
+                        });
+                    }
+
                     return { 
                         success: true, 
                         result: toolResult,

package/pathways/system/entity/tools/sys_tool_browser.js

@@ -12,7 +12,7 @@ export default {
         icon: "🌍",
         function: {
             name: "WebPageContent",
-            description: "This tool allows you to fetch and extract the text content from any webpage. Use this when you need to analyze or understand the content of a specific webpage.",
+            description: "This tool allows you to fetch and extract the text content and a screenshot if requested from any webpage. Use this when you need to analyze or understand the content of a specific webpage.",
             parameters: {
                 type: "object",
                 properties: {
@@ -20,6 +20,10 @@ export default {
                         type: "string",
                         description: "The complete URL of the webpage to fetch and analyze"
                     },
+                    takeScreenshot: {
+                        type: "boolean",
+                        description: "Whether to include a screenshot of the webpage in the response - slower, but can be helpful for digging deeper if the text content is not enough to answer the question"
+                    },
                     userMessage: {
                         type: "string",
                         description: "A user-friendly message that describes what you're doing with this tool"
@@ -54,7 +58,8 @@ export default {
                 searchResultId: getSearchResultId(),
                 title: "Webpage Content",
                 url: data.url,
-                content: data.text
+                content: data.text,
+                screenshot: args.takeScreenshot ? data.screenshot_base64 : undefined
             };
 
             resolver.tool = JSON.stringify({ toolUsed: "WebPageContent" });

package/pathways/system/entity/tools/sys_tool_browser_jina.js

@@ -0,0 +1,71 @@
+// sys_tool_browser.js
+// Tool pathway that handles web page content scraping functionality
+import logger from '../../../../lib/logger.js';
+import { config } from '../../../../config.js';
+import { getSearchResultId } from '../../../../lib/util.js';
+
+export default {
+    prompt: [],
+    timeout: 300,
+    toolDefinition: { 
+        type: "function",
+        icon: "🌎",
+        function: {
+            name: "WebPageContentJina",
+            description: "This tool allows you to fetch and extract the text content from any webpage using the Jina API. This is a great fallback for web page content if you don't get a good enough response from your other browser tool.",
+            parameters: {
+                type: "object",
+                properties: {
+                    url: {
+                        type: "string",
+                        description: "The complete URL of the webpage to fetch and analyze"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["url", "userMessage"]
+            }
+        }
+    },
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        // Check if browser service URL is available
+        const jinaApiKey = config.get('jinaApiKey');
+        if (!jinaApiKey) {
+            throw new Error("Jina API key is not available - missing JINA_API_KEY configuration");
+        }
+
+        try {
+            const scrapeUrl = `https://r.jina.ai/${encodeURIComponent(args.url)}`;
+            const token = `Bearer ${jinaApiKey}`;
+
+            const response = await fetch(scrapeUrl, {
+            headers: {
+                'Authorization': token
+            }
+            });
+
+            if (!response.ok) {
+                throw new Error(`Browser service returned error: ${response.status} ${response.statusText}`);
+            }
+
+            const data = await response.text();
+            
+            // Create a result object with the scraped content
+            const result = {
+                searchResultId: getSearchResultId(),
+                title: "Webpage Content",
+                url: args.url,
+                content: data
+            };
+
+            resolver.tool = JSON.stringify({ toolUsed: "WebPageContent" });
+            return JSON.stringify({ _type: "SearchResponse", value: [result] });
+        } catch (e) {
+            logger.error(`Error in browser tool: ${e}`);
+            throw e;
+        }
+    }
+}; 

package/server/plugins/modelPlugin.js

@@ -549,8 +549,20 @@ class ModelPlugin {
             cortexRequest.cache = config.get('enableCache') && (pathway.enableCache || pathway.temperature == 0);
             this.logRequestStart();
 
-            const { data: responseData, duration: requestDuration } = await executeRequest(cortexRequest);
+            const response = await executeRequest(cortexRequest);
             
+            // Add null check and default values for response
+            if (!response) {
+                throw new Error('Request failed - no response received');
+            }
+
+            const { data: responseData, duration: requestDuration } = response;
+            
+            // Validate response data
+            if (!responseData) {
+                throw new Error('Request failed - no data in response');
+            }
+
             const errorData = Array.isArray(responseData) ? responseData[0] : responseData;
             if (errorData && errorData.error) {
                 const newError = new Error(errorData.error.message);
@@ -558,21 +570,32 @@ class ModelPlugin {
                 throw newError;
             }
         
-            this.logAIRequestFinished(requestDuration);
+            this.logAIRequestFinished(requestDuration || 0);
             const parsedData = this.parseResponse(responseData);
             this.logRequestData(data, parsedData, prompt);
 
             return parsedData;
         } catch (error) {
-            // Log the error and continue
+            // Enhanced error logging
             const errorMessage = error?.response?.data?.message
                                  ?? error?.response?.data?.error?.message
                                  ?? error?.message
-                                 ?? String(error); // Fallback to string representation
+                                 ?? String(error);
+            
+            // Log the full error details for debugging
             logger.error(`Error in executeRequest for ${this.pathwayName}: ${errorMessage}`);
+            if (error.response) {
+                logger.error(`Response status: ${error.response.status}`);
+                logger.error(`Response headers: ${JSON.stringify(error.response.headers)}`);
+            }
             if (error.data) {
                 logger.error(`Additional error data: ${JSON.stringify(error.data)}`);
             }
+            if (error.stack) {
+                logger.error(`Error stack: ${error.stack}`);
+            }
+
+            // Throw a more informative error
             throw new Error(`Execution failed for ${this.pathwayName}: ${errorMessage}`);
         }
     }

package/server/plugins/openAiVisionPlugin.js

@@ -44,7 +44,7 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
                             if (typeof parsedItem === 'object' && parsedItem !== null) {
                                 // Handle both 'image' and 'image_url' types
                                 if (parsedItem.type === 'image' || parsedItem.type === 'image_url') {
-                                    const url = parsedItem.image_url?.url || parsedItem.url;
+                                    const url = parsedItem.url || parsedItem.image_url?.url;
                                     if (url && await this.validateImageUrl(url)) {
                                         return { type: 'image_url', image_url: { url } };
                                     }