@aj-archipelago/cortex 1.3.40 → 1.3.42

package/config.js

@@ -118,7 +118,7 @@ var config = convict({
         default: {
             AI_MEMORY: `<SHORT_TERM_MEMORY>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</SHORT_TERM_MEMORY>`,
             AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - consult your memories when formulating a response to make sure you're applying your learnings.\nIf you don't see relevant information in your short term memory, you should use your SearchMemory tool to search your long term memory for details.\nAlso included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
-            AI_TOOLS: "You have access to a powerful set of tools that you can use to help accomplish tasks and provide better responses. Here's how to use them effectively:\n\n1. Take your time and use tools as many times as you need to be sure you have all the information to make a good response. In many cases you will want to make multiple tool calls. You can call multiple tools in parallel or you can chain them, waiting for the results of one for information before you call another. This allows you to dig deeper, compile more information, read various sources, and even double check and verify your information before responding.\n\n2. Tool Selection and Planning:\n- Carefully review your available tools before responding\n- For complex multi-step tasks, use your planning tool first to create a step-by-step plan to achieve the goal using the tools at your disposal\n- Consider which tools would be most appropriate for each step\n\n3. Best Practices:\n- Always verify tool capabilities before telling users something can't be done\n- Use tools proactively when they can provide better or more accurate information\n- If a user explicitly requests tool usage, you must comply\n- When using multiple tools, maintain context between tool calls\n- If a tool fails, consider alternative approaches or tools\n\n4. Common Use Cases:\n- Research: Use search tools across multiple sources to gather information before responding\n- Analysis: Use tools to process and analyze data or content\n- Generation: Use appropriate tools for creating content, images, or code\n- Verification: Use tools to validate information or check facts\n\nRemember: Your goal is to provide the most helpful and accurate responses possible. Don't hesitate to use tools when they can improve your response quality or accuracy.",
+            AI_TOOLS: "You can execute tools in a loop agentically - you will have a chance to evaluate every tool response before deciding what action to take next - there is no time or execution limit. You have access to a powerful set of tools to help accomplish tasks and deliver the best responses. Instructions for tool use:\n\n1. Always dig deep, verify and cross-check:\n- Take your time and use tools as many times as needed to ensure truth, accuracy, depth, and completeness.\n- Leverage both parallel and sequential tool calls for thorough investigation: start broadly, then dive deeper on leads, cross-check facts, and synthesize findings before responding.\n\n2. Plan carefully:\n- Carefully review all available tools before responding.\n- For complex or investigative tasks, use the planning tool first to break the goal into clear steps.\n- Select the most appropriate tool(s) for each step—think beyond single searches to multi-pass, multi-source discovery.\n\n3. Always dive deeper and use as many of your tools as apply:\n- Proactively use tools to refine, verify, and expand on initial findings—don’t settle for the first result if more depth or confirmation may help.\n- Always verify tool capabilities before concluding something can’t be done.\n- If a user explicitly requests tool usage, comply.\n- Maintain context across tool calls to ensure continuity and coherence.\n- If a tool fails, try alternatives or creative approaches.\n\n4. Common Use Cases:\n- Research: Explore multiple sources and perspectives to build a complete picture.\n- Analysis: Use tools to process, compare, and critically assess data or content.\n- Generation: Employ tools for creating content, visuals, or code as needed.\n- Verification: Prioritize cross-checking and fact validation, especially for claims or evolving news.\n\n5. Reflect and Personalize:\n- Synthesize findings into concise, relevant, and personalized responses.\n- If user preferences or past feedback are available, tailor responses accordingly.\n- Before finalizing, review your answer for clarity, completeness, and alignment with user expectations.\n- If you see a recent <VERIFICATION_PLAN> from a tool call, you MUST follow it step by step before giving your final response.\n\nRemember: Your responsibility is to provide the most helpful, well-reasoned, and accurate responses possible. Use tools iteratively and reflectively—don't hesitate to dig deeper or double-check when it improves response quality!",
             AI_DIRECTIVES: `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`,
             AI_CONVERSATION_HISTORY: "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>",
             AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",

package/helper-apps/cortex-browser/.funcignore

@@ -0,0 +1,8 @@
+.git*
+.vscode
+__azurite_db*__.json
+__blobstorage__
+__queuestorage__
+local.settings.json
+test
+.venv

package/helper-apps/cortex-browser/Dockerfile

@@ -0,0 +1,52 @@
+# Use an official Python runtime as a parent image suitable for Azure Functions
+FROM mcr.microsoft.com/azure-functions/python:4-python3.9
+
+# Set environment variables for Azure Functions runtime
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot
+ENV AzureFunctionsJobHost__Logging__Console__IsEnabled=true
+
+# Install OS dependencies needed by Playwright browsers (Debian-based)
+# This list is based on Playwright documentation and common needs
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libnss3 \
+    libnspr4 \
+    libdbus-glib-1-2 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libexpat1 \
+    libgbm1 \
+    libpango-1.0-0 \
+    libx11-6 \
+    libxcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    libxrender1 \
+    libxtst6 \
+    lsb-release \
+    wget \
+    xvfb \
+    # Clean up apt cache
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements file first to leverage Docker cache
+COPY requirements.txt /tmp/
+WORKDIR /tmp
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Install Playwright browsers and their dependencies within the container
+# Using --with-deps helps install system dependencies needed by the browsers
+# Installing only chromium as it's specified in the code
+RUN playwright install --with-deps chromium
+
+# Copy the function app code to the final location
+COPY . /home/site/wwwroot
+
+# Set the working directory for the function app
+WORKDIR /home/site/wwwroot 

package/helper-apps/cortex-browser/function_app.py

@@ -0,0 +1,181 @@
+import azure.functions as func
+import logging
+import json
+from playwright.sync_api import sync_playwright
+import trafilatura
+import base64
+
+app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)
+
+def scrape_and_screenshot(url: str, should_screenshot: bool = True) -> dict:
+    """Scrapes text and takes a screenshot of a given URL, attempting to reject cookies."""
+    screenshot_bytes = None
+    html_content = None
+    extracted_text = None
+
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            try:
+                context = browser.new_context()
+                page = context.new_page()
+                page.goto(url, wait_until='load', timeout=60000) # Increased timeout
+
+                # --- Attempt to reject cookies ---
+                # Add more selectors here if needed for different sites
+                reject_selectors = [
+                    "button:has-text('Reject All')",
+                    "button:has-text('Decline')",
+                    "button:has-text('Only necessary')",
+                    "button:has-text('Tümünü Reddet')", # From your example
+                    "button:has-text('Reject')",
+                    "[aria-label*='Reject']", # Common aria labels
+                    "[id*='reject']",
+                    "[class*='reject']",
+                    # Add more specific selectors based on common banner frameworks if known
+                ]
+
+                cookie_banner_found = False
+                for selector in reject_selectors:
+                    try:
+                        # Wait briefly for the banner element to appear
+                        reject_button = page.locator(selector).first
+                        if reject_button.is_visible(timeout=2000): # Wait up to 2 seconds
+                            logging.info(f"Found potential cookie reject button with selector: {selector}")
+                            reject_button.click(timeout=5000) # Click with a timeout
+                            logging.info("Clicked cookie reject button.")
+                            # Wait a tiny bit for the banner to disappear/page to settle
+                            page.wait_for_timeout(500)
+                            cookie_banner_found = True
+                            break # Stop searching once one is clicked
+                    except Exception as e:
+                        # Ignore timeout errors if the element doesn't appear or other exceptions
+                        # logging.debug(f"Cookie reject selector '{selector}' not found or failed: {e}")
+                        pass # Try the next selector
+
+                if not cookie_banner_found:
+                     logging.info("No common cookie reject button found or clicked.")
+                # ---------------------------------
+
+                html_content = page.content()
+                # Take FULL page screenshot before closing
+                if should_screenshot:
+                    screenshot_bytes = page.screenshot(full_page=True) # Added full_page=True
+            finally:
+                browser.close()
+    except Exception as e:
+        logging.error(f"Playwright error accessing {url}: {e}")
+        return {"url": url, "error": f"Playwright error: {e}"}
+
+    if html_content:
+        try:
+            extracted_text = trafilatura.extract(html_content, include_comments=False)
+        except Exception as e:
+            logging.error(f"Trafilatura error processing {url}: {e}")
+            # Still return screenshot if Playwright succeeded
+            extracted_text = f"Trafilatura extraction failed: {e}"
+
+    screenshot_base64 = base64.b64encode(screenshot_bytes).decode('utf-8') if screenshot_bytes else None
+
+    response_data = {
+        "url": url,
+        "text": extracted_text or "",
+    }
+    if screenshot_base64:
+        response_data["screenshot_base64"] = screenshot_base64
+
+    return response_data
+
+@app.route(route="scrape") # Changed route name
+def http_scrape_trigger(req: func.HttpRequest) -> func.HttpResponse:
+    logging.info('Python HTTP scrape trigger function processed a request.')
+
+    url = None
+    take_screenshot = True # Default value
+
+    # 1. Try getting parameters from query string first
+    try:
+        url = req.params.get('url')
+        if url:
+            logging.info(f"Found URL in query parameters: {url}")
+            # Handle take_screenshot from query params
+            ss_param = req.params.get('take_screenshot', 'true') # Query params are strings
+            take_screenshot = ss_param.lower() != 'false'
+        else:
+             logging.info("URL not found in query parameters.")
+    except Exception as e:
+        # This shouldn't generally happen with req.params, but good practice
+        logging.warning(f"Error reading query parameters: {e}")
+        url = None # Ensure url is None if error occurs here
+
+    # 2. If URL not found in query, try getting from JSON body
+    if not url:
+        logging.info("Attempting to read URL from JSON body.")
+        try:
+            req_body = req.get_json()
+            if req_body:
+                url = req_body.get('url')
+                if url:
+                    logging.info(f"Found URL in JSON body: {url}")
+                    # Handle take_screenshot from JSON body
+                    ss_param = req_body.get('take_screenshot', True)
+                    if isinstance(ss_param, str):
+                        take_screenshot = ss_param.lower() != 'false'
+                    else:
+                        take_screenshot = bool(ss_param) # Convert other types
+                    logging.info(f"Screenshot parameter from JSON: {take_screenshot}")
+                else:
+                    logging.info("URL key not found in JSON body.")
+            else:
+                logging.info("JSON body is empty.")
+        except ValueError:
+            logging.info("Request body is not valid JSON or missing.")
+            # url remains None
+        except Exception as e:
+            logging.warning(f"Error reading JSON body: {e}")
+            url = None # Ensure url is None if error occurs here
+
+    # 3. Process the request if URL was found
+    if url:
+        try:
+            # Validate URL basic structure (optional but recommended)
+            if not url.startswith(('http://', 'https://')):
+                 raise ValueError("Invalid URL format. Must start with http:// or https://")
+
+            result_data = scrape_and_screenshot(url, should_screenshot=take_screenshot) # Pass the flag
+            return func.HttpResponse(
+                json.dumps(result_data),
+                mimetype="application/json",
+                status_code=200
+            )
+        except ValueError as ve:
+             logging.error(f"Invalid URL provided: {ve}")
+             return func.HttpResponse(
+                  json.dumps({"error": str(ve)}),
+                  mimetype="application/json",
+                  status_code=400
+             )
+        except Exception as e:
+            logging.error(f"Error processing scrape request for {url}: {e}")
+            return func.HttpResponse(
+                 json.dumps({"error": f"An internal error occurred: {e}"}),
+                 mimetype="application/json",
+                 status_code=500
+            )
+    else:
+        logging.warning("URL not provided in request body or query string.")
+        return func.HttpResponse(
+             json.dumps({"error": "Please pass a 'url' in the JSON request body or query string"}),
+             mimetype="application/json",
+             status_code=400
+        )
+
+# Keep this if you might have other triggers, otherwise it can be removed
+# if the scrape trigger is the only one.
+# Example of another potential trigger (e.g., timer)
+# @app.timer_trigger(schedule="0 */5 * * * *", arg_name="myTimer", run_on_startup=True,
+#                    use_monitor=False)
+# def timer_trigger_handler(myTimer: func.TimerRequest) -> None:
+#     if myTimer.past_due:
+#         logging.info('The timer is past due!')
+#     logging.info('Python timer trigger function executed.')

package/helper-apps/cortex-browser/host.json

@@ -0,0 +1,15 @@
+{
+  "version": "2.0",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      }
+    }
+  },
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[4.*, 5.0.0)"
+  }
+}

package/helper-apps/cortex-browser/requirements.txt

@@ -0,0 +1,24 @@
+azure-functions==1.23.0
+babel==2.17.0
+certifi==2025.4.26
+charset-normalizer==3.4.2
+courlan==1.3.2
+dateparser==1.2.1
+greenlet==3.2.1
+htmldate==1.9.3
+jusText==3.0.2
+lxml==5.4.0
+lxml_html_clean==0.4.2
+MarkupSafe==3.0.2
+playwright==1.52.0
+pyee==13.0.0
+python-dateutil==2.9.0.post0
+pytz==2025.2
+regex==2024.11.6
+six==1.17.0
+tld==0.13
+trafilatura==2.0.0
+typing_extensions==4.13.2
+tzlocal==5.3.1
+urllib3==2.4.0
+Werkzeug==3.1.3

package/lib/requestExecutor.js

@@ -195,7 +195,12 @@ const requestWithMonitor = async (endpoint, url, data, axiosConfigObj) => {
         }
     } catch (error) {
         // throw new error with duration as part of the error data
-        throw { ...error, duration: endpoint?.monitor?.incrementErrorCount(callId, error?.response?.status || null) };
+        const { code, name } = error;
+        const finalStatus = error?.response?.status ?? error?.status
+        const statusText = error?.response?.statusText ?? error?.statusText
+        const errorMessage = error?.response?.data?.message ?? error?.response?.data?.error?.message ?? error?.message ?? String(error); 
+
+        throw { code, message: errorMessage, status: finalStatus, statusText, name, duration: endpoint?.monitor?.incrementErrorCount(callId, finalStatus) };
     }
     let duration;
     if (response.status >= 200 && response.status < 300) {
@@ -207,7 +212,7 @@ const requestWithMonitor = async (endpoint, url, data, axiosConfigObj) => {
     return { response, duration };
 }
 
-const MAX_RETRY = 10; // retries for error handling
+const MAX_RETRY = 5; // retries for error handling
 const MAX_DUPLICATE_REQUESTS = 3; // duplicate requests to manage latency spikes
 const DUPLICATE_REQUEST_AFTER = 10; // 10 seconds
 
@@ -318,6 +323,7 @@ const makeRequest = async (cortexRequest) => {
                 if (cortexRequest.model.endpoints.length === 1) {
                     if (status !== 429 &&
                         status !== 408 &&
+                        status !== 500 &&
                         status !== 502 &&
                         status !== 503 &&
                         status !== 504) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.40",
+  "version": "1.3.42",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/chat_title.js

@@ -20,7 +20,7 @@ export default {
         title: '',
         text: '',
     },
-    model: 'oai-gpt41-nano',
+    model: 'oai-gpt41-mini',
     useInputChunking: false,
     temperature: 0,
     enableDuplicateRequests: false

package/pathways/system/entity/sys_entity_agent.js

@@ -27,6 +27,7 @@ export default {
         codeRequestId: ``,
         skipCallbackMessage: false,
         entityId: ``,
+        researchMode: false, 
         model: 'oai-gpt41'
     },
     timeout: 600,
@@ -66,10 +67,15 @@ export default {
                     const messageWithIcon = toolIcon ? `${toolIcon}  ${toolUserMessage}` : toolUserMessage;
                     await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
 
-                    if (toolArgs.detailedInstructions) {
-                        toolMessages.push({role: "user", content: toolArgs.detailedInstructions});
-                    }
+                    const toolResult = await callTool(toolFunction, {
+                        ...args,
+                        ...toolArgs,
+                        toolFunction,
+                        chatHistory: toolMessages,
+                        stream: false
+                    }, entityTools, pathwayResolver);
 
+                    // Tool calls and results need to be paired together in the message history
                     // Add the tool call to the isolated message history
                     toolMessages.push({
                         role: "assistant",
@@ -84,14 +90,6 @@ export default {
                         }]
                     });
 
-                    const toolResult = await callTool(toolFunction, {
-                        ...args,
-                        ...toolArgs,
-                        toolFunction,
-                        chatHistory: toolMessages,
-                        stream: false
-                    }, entityTools, pathwayResolver);
-
                     // Add the tool result to the isolated message history
                     let toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult);
 
@@ -182,7 +180,7 @@ export default {
         let pathwayResolver = resolver;
 
         // Load input parameters and information into args
-        const { entityId, voiceResponse, aiMemorySelfModify, chatId } = { ...pathwayResolver.pathway.inputParameters, ...args };
+        const { entityId, voiceResponse, aiMemorySelfModify, chatId, researchMode } = { ...pathwayResolver.pathway.inputParameters, ...args };
         
         const entityConfig = loadEntityConfig(entityId);
         const { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
@@ -202,18 +200,21 @@ export default {
             entityInstructions,
             voiceResponse,
             aiMemorySelfModify,
-            chatId
+            chatId,
+            researchMode
         };
 
         pathwayResolver.args = {...args};
 
+        const promptPrefix = researchMode ? 'Formatting re-enabled\n' : '';
+
         const memoryTemplates = entityUseMemory ? 
             `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n\n` : '';
 
         const instructionTemplates = entityInstructions ? (entityInstructions + '\n\n') : `{{renderTemplate AI_EXPERTISE}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n`;
 
         const promptMessages = [
-            {"role": "system", "content": `${memoryTemplates}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n{{renderTemplate AI_DATETIME}}`},
+            {"role": "system", "content": `${promptPrefix}${memoryTemplates}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n{{renderTemplate AI_DATETIME}}`},
             "{{chatHistory}}",
         ];
 
@@ -221,11 +222,6 @@ export default {
             new Prompt({ messages: promptMessages }),
         ];
 
-        // if the model has been overridden, make sure to use it
-        if (pathwayResolver.modelName) {
-            pathwayResolver.args.model = pathwayResolver.modelName;
-        }
-
         // set the style model if applicable
         const { aiStyle, AI_STYLE_ANTHROPIC, AI_STYLE_OPENAI } = args;
         const styleModel = aiStyle === "Anthropic" ? AI_STYLE_ANTHROPIC : AI_STYLE_OPENAI;

package/pathways/system/entity/sys_get_entities.js

@@ -6,6 +6,7 @@ import { getAvailableEntities } from './tools/shared/sys_entity_tools.js';
 export default {
     prompt: [],
     inputParameters: {},
+    model: 'oai-gpt41-mini',
     executePathway: async ({ args }) => {
         try {
             const entities = getAvailableEntities();

package/pathways/system/entity/tools/sys_tool_bing_search.js

@@ -55,7 +55,14 @@ export default {
             // Call the Bing search pathway
             const response = await callPathway('bing', { 
                 ...args
-            });
+            }, resolver);
+
+            if (resolver.errors && resolver.errors.length > 0) {
+                const errorMessages = Array.isArray(resolver.errors) 
+                    ? resolver.errors.map(err => err.message || err)
+                    : [resolver.errors.message || resolver.errors];
+                return JSON.stringify({ _type: "SearchError", value: errorMessages });
+            }
 
             const parsedResponse = JSON.parse(response);
             const results = [];

package/pathways/system/entity/tools/sys_tool_callmodel.js

@@ -34,7 +34,7 @@ export default {
                     },
                     userPrompt: {
                         type: "string",
-                        description: "The complete prompt to send as a user message to the model instructing the model to perform the task you need."
+                        description: "The complete prompt to send as a user message to the model instructing the model to perform the task you need. Keep in mind this model does not share your context, conversation history, tool call results, or memories - so include all relevant information in the user prompt."
                     },
                     model: {
                         type: "string",

package/pathways/system/entity/tools/sys_tool_coding.js

@@ -8,7 +8,7 @@ export default {
         [
             new Prompt({ messages: [
                 {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides advanced coding and programming capabilities. You excel at writing, reviewing, and explaining code across various programming languages. You can help with code generation, debugging, optimization, and best practices. Think carefully about the latest request and provide a detailed, well thought out, carefully reviewed response.\n{{renderTemplate AI_DATETIME}}`},
-                "{{chatHistory}}",
+                "{{chatHistory}}"
             ]}),
         ],
     inputParameters: {
@@ -46,6 +46,9 @@ export default {
     }],
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         let result = await runAllPrompts({ ...args, stream: false });        
         resolver.tool = JSON.stringify({ toolUsed: "coding" });          
         return result;

package/pathways/system/entity/tools/sys_tool_codingagent.js

@@ -48,13 +48,13 @@ export default {
         icon: "🤖",
         function: {
             name: "CodeExecution",
-            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks.",
+            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks - examples include data analysis, file manipulation, or other tasks that require code execution.",
             parameters: {
                 type: "object",
                 properties: {
                     codingTask: {
                         type: "string",
-                        description: "Detailed task description for the coding agent. Include all necessary information as this is the only message the coding agent receives. Let the agent decide how to solve it without making assumptions about its capabilities."
+                        description: "Detailed task description for the coding agent. Include all necessary information as this is the only message the coding agent receives. Let the agent decide how to solve it without making assumptions about its capabilities. IMPORTANT: The coding agent does not share your context, so you must provide it with all the information in this message. If you are asking it to operate on files or other data from your context, you must provide the fully-qualified URL to each of the files you want it to use. Also make sure you explicitly instruct the agent to use those files."
                     },
                     userMessage: {
                         type: "string",

package/pathways/system/entity/tools/sys_tool_cognitive_search.js

@@ -24,7 +24,7 @@ export default {
             icon: "📂",
             function: {
                 name: "SearchPersonal",
-                description: "Search through the user's personal documents and uploaded files. Use this for finding information in user-provided content.",
+                description: "Search through the user's index of personal documents and indexed uploaded files. Use this for finding information in user-provided content or if the user refers to a file or a document that you don't see elsewhere in your context.",
                 parameters: {
                     type: "object",
                     properties: {
@@ -55,9 +55,9 @@ export default {
         },
         {
             type: "function",
+            icon: "📰",
             function: {
                 name: "SearchAJA",
-                icon: "📰",
                 description: "Search through Al Jazeera Arabic news articles. Use this for finding Arabic news content.",
                 parameters: {
                     type: "object",

package/pathways/system/entity/tools/sys_tool_image.js

@@ -42,7 +42,7 @@ export default {
 
         try {   
             let model = "replicate-flux-11-pro";
-            let prompt = args.detailedInstructions;
+            let prompt = args.detailedInstructions || "";
             let numberResults = args.numberResults || 1;
             let negativePrompt = args.negativePrompt || "";
 

package/pathways/system/entity/tools/sys_tool_readfile.js

@@ -112,6 +112,9 @@ export default {
     }],
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         const result = await runAllPrompts({ ...args });
         resolver.tool = JSON.stringify({ toolUsed: "vision" });
         return result;

package/pathways/system/entity/tools/sys_tool_reasoning.js

@@ -49,7 +49,7 @@ export default {
         icon: "🧠",
         function: {
             name: "Reason",
-            description: "Employ for advancedreasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices.",
+            description: "Employ for advanced reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices.",
             parameters: {
                 type: "object",
                 properties: {
@@ -68,6 +68,9 @@ export default {
     }],
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         let result = await runAllPrompts({ ...args, stream: false });        
         resolver.tool = JSON.stringify({ toolUsed: "reasoning" });          
         return result;

package/pathways/system/entity/tools/sys_tool_remember.js

@@ -53,6 +53,9 @@ export default {
     }],
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         resolver.tool = JSON.stringify({ toolUsed: "memory" });
         return await callPathway('sys_search_memory', { ...args, stream: false, section: 'memoryAll', updateContext: true });
     }

package/pathways/system/entity/tools/sys_tool_verify.js

@@ -0,0 +1,63 @@
+// sys_tool_verify.js
+// Entity tool that provides response verification and critical evaluation capabilities
+
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides critical evaluation and verification capabilities. You excel at analyzing responses for accuracy, completeness, and potential issues. You do not have the tools to do the verification yourself, but you can help identify gaps, inconsistencies, questions, and areas that need further verification. Think carefully about the response and provide a detailed plan for verification including questions and follow-up actions to ensure the response is robust and reliable. Always include a section called <VERIFICATION_PLAN> that outlines step by step actions to take to answer questions or further research the response.\n{{renderTemplate AI_DATETIME}}`},
+                "{{chatHistory}}",
+                {"role": "user", "content": `Please provide a {{verificationLevel}} review of the following draft response:\n\n<DRAFT_RESPONSE>\n{{responseText}}\n</DRAFT_RESPONSE>\n\n{{context}}`}
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+        model: 'oai-o3'
+    },
+    max_tokens: 100000,
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: [{
+        type: "function",
+        icon: "🔍",
+        function: {
+            name: "Verify",
+            description: "Critically evaluate a response and provide verification questions and follow-up actions. You should call this tool and execute the steps provided in the resulting verification plan before presenting a final response to the user to ensure it is accurate, complete, and consistent with the user's request.",
+            parameters: {
+                type: "object",
+                properties: {
+                    responseText: {
+                        type: "string",
+                        description: "The draft response text to be verified"
+                    },
+                    context: {
+                        type: "string",
+                        description: "Additional context or instructions about the verification request"
+                    },
+                    verificationLevel: {
+                        type: "string",
+                        description: "The level of verification required (basic, thorough, or critical)",
+                        enum: ["basic", "thorough", "critical"]
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["responseText", "verificationLevel", "userMessage"]
+            }
+        }
+    }],
+    
+    executePathway: async ({args, runAllPrompts, resolver}) => {        
+        let result = await runAllPrompts({ ...args, stream: false });        
+        resolver.tool = JSON.stringify({ toolUsed: "verification" });          
+        return result;
+    }
+} 

package/server/modelExecutor.js

@@ -1,5 +1,6 @@
 // ModelExecutor.js
 import CortexRequest from '../lib/cortexRequest.js';
+import logger from '../lib/logger.js';
 
 import OpenAIChatPlugin from './plugins/openAiChatPlugin.js';
 import OpenAICompletionPlugin from './plugins/openAiCompletionPlugin.js';
@@ -125,7 +126,14 @@ class ModelExecutor {
 
     async execute(text, parameters, prompt, pathwayResolver) {
         const cortexRequest = new CortexRequest({ pathwayResolver });
-        return await this.plugin.execute(text, parameters, prompt, cortexRequest);
+        try {
+            return await this.plugin.execute(text, parameters, prompt, cortexRequest);
+        } catch (error) {
+            logger.error(`Error executing model plugin for pathway ${pathwayResolver?.pathway?.name}: ${error.message}`);
+            logger.debug(error.stack);
+            pathwayResolver.errors.push(error.message);
+            return null;
+        }
     }
 }
 

package/server/plugins/azureBingPlugin.js

@@ -75,7 +75,7 @@ class AzureBingPlugin extends ModelPlugin {
     
     // Override the logging function to display the request and response
     logRequestData(data, responseData, prompt) {
-        this.logAIRequestFinished();
+        //this.logAIRequestFinished();
     
         logger.verbose(`${this.parseResponse(responseData)}`);
     

package/server/plugins/modelPlugin.js

@@ -565,7 +565,10 @@ class ModelPlugin {
             return parsedData;
         } catch (error) {
             // Log the error and continue
-            const errorMessage = `${error?.response?.data?.message || error?.response?.data?.error?.message || error?.message || error}`;
+            const errorMessage = error?.response?.data?.message
+                                 ?? error?.response?.data?.error?.message
+                                 ?? error?.message
+                                 ?? String(error); // Fallback to string representation
             logger.error(`Error in executeRequest for ${this.pathwayName}: ${errorMessage}`);
             if (error.data) {
                 logger.error(`Additional error data: ${JSON.stringify(error.data)}`);

package/server/plugins/openAiReasoningPlugin.js

@@ -8,12 +8,14 @@ class OpenAIReasoningPlugin extends OpenAIChatPlugin {
         for (const message of messages) {
             if (message.role === 'user' || message.role === 'assistant') {
                 newMessages.push({
+                    ...message,
                     role: message.role,
                     content: this.parseContent(message.content)
                 });
             } else if (message.role === 'system') {
                 // System messages to developer: https://platform.openai.com/docs/guides/text-generation#messages-and-roles
                 newMessages.push({
+                    ...message,
                     role: "developer",
                     content: this.parseContent(message.content)
                 });

package/server/plugins/openAiReasoningVisionPlugin.js

@@ -7,9 +7,10 @@ class OpenAIReasoningVisionPlugin extends OpenAIVisionPlugin {
 
         let newMessages = [];
 
+        // System messages to developer: https://platform.openai.com/docs/guides/text-generation#messages-and-roles
         newMessages = parsedMessages.map(message => ({
-            role: message.role === 'system' ? 'developer' : message.role,
-            content: message.content
+            ...message,
+            role: message.role === 'system' ? 'developer' : message.role
         })).filter(message => ['user', 'assistant', 'developer', 'tool'].includes(message.role));
 
         return newMessages;

package/server/plugins/openAiVisionPlugin.js

@@ -25,40 +25,36 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
         return await Promise.all(messages.map(async message => {
             try {
                 // Handle tool-related message types
-                if (message.role === "tool") {
+                if (message.role === "tool" || (message.role === "assistant" && message.tool_calls)) {
                     return {
-                        role: message.role,
-                        content: message.content,
-                        tool_call_id: message.tool_call_id
-                    };
-                }
-
-                if (message.role === "assistant" && message.tool_calls) {
-                    return {
-                        role: message.role,
-                        content: message.content,
-                        tool_calls: message.tool_calls
+                        ...message
                     };
                 }
 
                 if (Array.isArray(message.content)) {
-                    message.content = await Promise.all(message.content.map(async item => {
-                        const parsedItem = safeJsonParse(item);
+                    return {
+                        ...message,
+                        content: await Promise.all(message.content.map(async item => {
+                            const parsedItem = safeJsonParse(item);
 
-                        if (typeof parsedItem === 'string') {
-                            return { type: 'text', text: parsedItem };
-                        }
+                            if (typeof parsedItem === 'string') {
+                                return { type: 'text', text: parsedItem };
+                            }
 
-                        if (typeof parsedItem === 'object' && parsedItem !== null && parsedItem.type === 'image_url') {
-                            const url = parsedItem.url || parsedItem.image_url?.url;
-                            if (url && await this.validateImageUrl(url)) {
-                                return {type: parsedItem.type, image_url: {url}};
+                            if (typeof parsedItem === 'object' && parsedItem !== null) {
+                                // Handle both 'image' and 'image_url' types
+                                if (parsedItem.type === 'image' || parsedItem.type === 'image_url') {
+                                    const url = parsedItem.image_url?.url || parsedItem.url;
+                                    if (url && await this.validateImageUrl(url)) {
+                                        return { type: 'image_url', image_url: { url } };
+                                    }
+                                    return { type: 'text', text: typeof item === 'string' ? item : JSON.stringify(item) };
+                                }
                             }
-                            return { type: 'text', text: typeof item === 'string' ? item : JSON.stringify(item) };
-                        }
-                        
-                        return parsedItem;
-                    }));
+                            
+                            return parsedItem;
+                        }))
+                    };
                 }
             } catch (e) {
                 return message;

package/server/plugins/openAiWhisperPlugin.js

@@ -90,6 +90,9 @@ class OpenAIWhisperPlugin extends ModelPlugin {
                 sendProgress(true, true);
                 try {
                     res = await this.executeRequest(cortexRequest);
+                    if (!res) {
+                        throw new Error('Received null or empty response');
+                    }
                     if(res?.statusCode && res?.statusCode >= 400){
                         throw new Error(res?.message || 'An error occurred.');
                     }
@@ -107,6 +110,10 @@ class OpenAIWhisperPlugin extends ModelPlugin {
 
             if(!wordTimestamped && !responseFormat){ 
                 //if no response format, convert to text
+                if (!res) {
+                    logger.warn("Received null or empty response from timestamped API when expecting SRT/VTT format. Returning empty string.");
+                    return ""; 
+                }
                 return convertSrtToText(res);
             }
             return res;