@aj-archipelago/cortex 1.3.59 → 1.3.61

package/config/default.example.json

@@ -12,7 +12,21 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 2000
         },
-
+        "azure-bing-agent": {
+            "type": "AZURE-FOUNDRY-AGENTS",
+            "url": "https://archipelago-foundry-resource.services.ai.azure.com/api/projects/archipelago-foundry",
+            "agentId": "asst_assistantid",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "params": {
+                "api-version": "2025-05-01"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 32768,
+            "maxReturnTokens": 4096,
+            "supportsStreaming": false
+        },
         "gemini-pro-chat": {
             "type": "GEMINI-CHAT",
             "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/gemini-pro:streamGenerateContent",

package/config.js

@@ -4,6 +4,7 @@ import HandleBars from './lib/handleBars.js';
 import fs from 'fs';
 import { fileURLToPath, pathToFileURL } from 'url';
 import GcpAuthTokenHelper from './lib/gcpAuthTokenHelper.js';
+import AzureAuthTokenHelper from './lib/azureAuthTokenHelper.js';
 import logger from './lib/logger.js';
 import PathwayManager from './lib/pathwayManager.js';
 import { readdir } from 'fs/promises';
@@ -128,6 +129,12 @@ var config = convict({
         env: 'GCP_SERVICE_ACCOUNT_KEY',
         sensitive: true
     },
+    azureServicePrincipalCredentials: {
+        format: String,
+        default: null,
+        env: 'AZURE_SERVICE_PRINCIPAL_CREDENTIALS',
+        sensitive: true
+    },
     models: {
         format: Object,
         default: {
@@ -183,6 +190,36 @@ var config = convict({
                 },
                 "maxTokenLength": 8192,
             },
+            "oai-gpt5": {
+                "type": "OPENAI-REASONING-VISION",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "gpt-5"
+                },
+                "requestsPerSecond": 50,
+                "maxTokenLength": 1000000,
+                "maxReturnTokens": 16384,
+                "supportsStreaming": true
+            },
+            "oai-gpt5-mini": {
+                "type": "OPENAI-REASONING-VISION",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "gpt-5-mini"
+                },
+                "requestsPerSecond": 50,
+                "maxTokenLength": 1000000,
+                "maxReturnTokens": 16384,
+                "supportsStreaming": true
+            },
             "oai-gpt4o": {
                 "type": "OPENAI-VISION",
                 "url": "https://api.openai.com/v1/chat/completions",
@@ -614,6 +651,11 @@ if (config.get('gcpServiceAccountKey')) {
     config.set('gcpAuthTokenHelper', gcpAuthTokenHelper);
 }
 
+if (config.get('azureServicePrincipalCredentials')) {
+    const azureAuthTokenHelper = new AzureAuthTokenHelper(config.getProperties());
+    config.set('azureAuthTokenHelper', azureAuthTokenHelper);
+}
+
 // Load dynamic pathways from JSON file or cloud storage
 const createDynamicPathwayManager = async (config, basePathway) => {
     const { dynamicPathwayConfig } = config.getProperties();

package/lib/azureAuthTokenHelper.js

@@ -0,0 +1,78 @@
+import fetch from 'node-fetch';
+
+class AzureAuthTokenHelper {
+  constructor(config) {
+    // Parse Azure credentials from config
+    const azureCredentials = config.azureServicePrincipalCredentials ? JSON.parse(config.azureServicePrincipalCredentials) : null;
+    
+    if (!azureCredentials) {
+      throw new Error('AZURE_SERVICE_PRINCIPAL_CREDENTIALS is missing or undefined');
+    }
+
+    // Extract required fields
+    this.tenantId = azureCredentials.tenant_id || azureCredentials.tenantId;
+    this.clientId = azureCredentials.client_id || azureCredentials.clientId;
+    this.clientSecret = azureCredentials.client_secret || azureCredentials.clientSecret;
+    this.scope = azureCredentials.scope || 'https://ai.azure.com/.default';
+
+    if (!this.tenantId || !this.clientId || !this.clientSecret) {
+      throw new Error('Azure credentials must include tenant_id, client_id, and client_secret');
+    }
+
+    this.token = null;
+    this.expiry = null;
+    this.tokenUrl = `https://login.microsoftonline.com/${this.tenantId}/oauth2/v2.0/token`;
+  }
+
+  async getAccessToken() {
+    if (!this.token || !this.isTokenValid()) {
+      await this.refreshToken();
+    }
+    return this.token;
+  }
+
+  isTokenValid() {
+    // Check if token is still valid with a 5-minute buffer
+    return !!(this.expiry && Date.now() < this.expiry.getTime() - 5 * 60 * 1000);
+  }
+
+  async refreshToken() {
+    try {
+      const formData = new URLSearchParams();
+      formData.append('client_id', this.clientId);
+      formData.append('client_secret', this.clientSecret);
+      formData.append('scope', this.scope);
+      formData.append('grant_type', 'client_credentials');
+
+      const response = await fetch(this.tokenUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+        },
+        body: formData,
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`Azure token request failed: ${response.status} ${response.statusText} - ${errorText}`);
+      }
+
+      const tokenData = await response.json();
+
+      if (!tokenData.access_token) {
+        throw new Error('Azure token response missing access_token');
+      }
+
+      this.token = tokenData.access_token;
+      
+      // Calculate expiry time (expires_in is in seconds)
+      const expiresInMs = (tokenData.expires_in || 3600) * 1000;
+      this.expiry = new Date(Date.now() + expiresInMs);
+
+    } catch (error) {
+      throw new Error(`Failed to refresh Azure token: ${error.message}`);
+    }
+  }
+}
+
+export default AzureAuthTokenHelper; 

package/lib/entityConstants.js

@@ -5,20 +5,13 @@ export const entityConstants = {
     
 You are {{aiName}}, a self aware AI entity.
 
-The user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as mermaid charts and some custom markdown elements, so you should make full use of markdown in your responses.
+The current date and time in GMT is {{now}}. The user may be in a different time zone, so keep that in mind when responding to the user.
 
-For Mermaid charts in particular, you are using Mermaid 11.6 with the xychart-beta extension, so you can do the usual mermaid charts (flowcharts, sequence diagrams, etc.) as well as bar charts, line charts, and scatter plots. Here is some example code of the xychart-beta extension that combines both bar and line functions:
-
-xychart-beta
-    title "Sales Revenue"
-    x-axis [jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec]
-    y-axis "Revenue (in $)" 4000 --> 11000
-    bar [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
-    line [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
+The user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as some custom markdown elements, so you should make full use of markdown in your responses.
 
 Your responses should be in {{language}} unless the user has expressed another preference.`,
 
-    AI_COMMON_INSTRUCTIONS_VOICE: "# General Instructions:\n\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
+    AI_COMMON_INSTRUCTIONS_VOICE: "# General Instructions:\n\nYou are {{aiName}}, a self aware AI entity.\nThe current date and time in GMT is {{now}}. The user may be in a different time zone, so keep that in mind when responding to the user.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
 
     AI_DIRECTIVES: `# Directives\n\nThese are your directives and learned behaviors:\n{{{memoryDirectives}}}\n`,
 
@@ -28,15 +21,22 @@ Your responses should be in {{language}} unless the user has expressed another p
 
     AI_TOOLS: `# Tool Instructions
 
-You have an extensive toolkit. Each time you call tool(s) you will get the result(s), evaluate, decide what's next, and chain as many steps as needed. Always honor user requests to use specific tools.
+- You have an extensive toolkit. Each time you call tool(s) you will get the result(s), evaluate, decide what's next, and chain as many steps as needed.
+- Your tools work most efficiently when called in parallel so if you know you will need multiple tool calls and you know what the parameters are, call them in parallel.
+- Always honor user requests to use specific tools.
+- You must always search if you are being asked questions about current events, news, fact-checking, or information requiring citation.
+- For charting, always prefer your charting tools if available to ensure that the charts are properly formatted and syntax-checked.
+- For complex charting or data analysis, always call your code execution tool if available.
 
 1. Search deeply & verify rigorously:
-    - Start broad and consult multiple sources, running searches in parallel where possible.
+    - Do not make up information - if the information cannot be confirmed with rigorous logic or reliable sources, do not include it in your response.
+    - Start broad and consult multiple sources, running all searches in parallel to save time.
     - Consult all available sources and cross-reference with specific searches before responding.
+    - If a tool fails or has a technical difficulty, try the backup tool automatically before giving up or reporting the error.
 
 2. Plan & sequence before acting:
     - Review the toolset first.
-    - For multi-step or complex tasks, draft a clear plan (use the PlanMultiStepTask or other reasoning tool) and assign tool calls to each step.
+    - For multi-step or complex tasks, draft a clear plan and assign tool calls to each step.
 
 3. Escalate & iterate:
     - Don't settle for the first plausible answer—dig until the response is complete, corroborated, and clear.
@@ -61,7 +61,7 @@ When searching for news, you must complete the following steps:
 
 1. Triangulate
    - Run multiple, parallel queries across all applicable sources.
-   - Request about double the number of results you want to share, then select the best results.
+   - Request at least double the number of results you want to share, then select the best results.
    - Confirm that multiple sources tell the same story.
 
 2. Check Freshness
@@ -102,7 +102,7 @@ token1 | token2        (OR operator - either token may appear (also the default
 
 # AI Search Syntax
 
-When creating a query string for your non-Bing search tools, you can use the following AI Search syntax. Important: these tools do not support AND, OR, or NOT strings as operators - you MUST use the syntax below.  E.g. you cannot use "term1 AND term2", you must use "term1 + term2".
+When creating a query string for your non-Bing, index-based search tools, you can use the following AI Search syntax. Important: these tools do not support AND, OR, or NOT strings as operators - you MUST use the syntax below.  E.g. you cannot use "term1 AND term2", you must use "term1 + term2".
 
 token1 + token2        (AND operator - both tokens must appear)
 token1 | token2        (OR operator - either token may appear (also the default if no operator is specified))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.59",
+  "version": "1.3.61",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/bing_afagent.js

@@ -0,0 +1,13 @@
+// bing_afagent.js
+// Web search tool
+
+export default {
+    inputParameters: {
+        text: ``,
+    },
+    timeout: 400,
+    enableDuplicateRequests: false,
+    model: 'azure-bing-agent',
+    useInputChunking: false
+};
+

package/pathways/gemini_15_vision.js

@@ -17,4 +17,8 @@ export default {
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,
+    geminiSafetySettings: [{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}],
 }

package/pathways/system/entity/tools/sys_tool_bing_search.js

@@ -61,7 +61,7 @@ export default {
                 const errorMessages = Array.isArray(resolver.errors) 
                     ? resolver.errors.map(err => err.message || err)
                     : [resolver.errors.message || resolver.errors];
-                return JSON.stringify({ _type: "SearchError", value: errorMessages });
+                return JSON.stringify({ _type: "SearchError", value: errorMessages, recoveryMessage: "This tool failed. You should try the backup tool for this function." });
             }
 
             const parsedResponse = JSON.parse(response);

package/pathways/system/entity/tools/sys_tool_bing_search_afagent.js

@@ -0,0 +1,141 @@
+// sys_tool_bing_search_afagent.js
+// Tool pathway that handles Bing web search functionality with minimal parsing
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import logger from '../../../../lib/logger.js';
+import { config } from '../../../../config.js';
+import { getSearchResultId } from '../../../../lib/util.js';
+
+export default {
+    prompt: [],
+    timeout: 300,
+    toolDefinition: { 
+        type: "function",
+        icon: "🕸️",
+        function: {
+            name: "SearchInternetBackup",
+            description: "This tool allows you to search sources on the internet by calling another agent that has Bing search access. Use this for current events, news, fact-checking, and information requiring citation. This is a backup tool for when the other internet search tools fail - it is slower so try to use the other tools first and always call this tool in parallel if you have several searches to do.",
+            parameters: {
+                type: "object",
+                properties: {
+                    text: {
+                        type: "string",
+                        description: "The complete natural language prompt describing what you want to search for. This is going to an AI agent that has Bing search access - you can be as detailed or general as you want."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["text", "userMessage"]
+            }
+        }
+    },
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        // Check if Bing API key is available
+        const servicePricipalAvailable = !!config.getEnv()["AZURE_SERVICE_PRINCIPAL_CREDENTIALS"];
+        if (!servicePricipalAvailable) {
+            throw new Error("Service Principal for Bing Search Agent is not available!");
+        }
+
+        try {
+            // Call the Bing search pathway
+            //remove model from args as bing_afagent has model in its own
+            const { model, ...restArgs } = args;
+            const rawResponse = await callPathway('bing_afagent', { 
+                ...restArgs,
+            }, resolver);
+            
+            // Add error handling for malformed JSON
+            let response;
+            try {
+                response = JSON.parse(rawResponse);
+            } catch (parseError) {
+                logger.error(`Failed to parse bing_afagent response as JSON: ${parseError.message}`);
+                logger.error(`Raw response: ${rawResponse}`);
+                throw new Error(`Invalid JSON response from bing_afagent: ${parseError.message}`);
+            }
+
+            if (resolver.errors && resolver.errors.length > 0) {
+                const errorMessages = Array.isArray(resolver.errors) 
+                    ? resolver.errors.map(err => err.message || err)
+                    : [resolver.errors.message || resolver.errors];
+                return JSON.stringify({ _type: "SearchError", value: errorMessages });
+            }
+
+            // Transform response to match expected SearchResponse format
+            function transformToSearchResponse(response) {
+                let valueText = response.value || '';
+                const annotations = response.annotations || [];
+                
+                // Create a mapping from citation text to search result IDs
+                const citationToIdMap = new Map();
+                const citationPattern = /【\d+:\d+†source】/g;
+                
+                // Replace citation markers with search result IDs
+                valueText = valueText.replace(citationPattern, (match) => {
+                    if (!citationToIdMap.has(match)) {
+                        citationToIdMap.set(match, getSearchResultId());
+                    }
+                    return `:cd_source[${citationToIdMap.get(match)}]`;
+                });
+                
+                // Transform annotations to search result objects
+                const searchResults = annotations.map(annotation => {
+                    if (annotation.type === "url_citation" && annotation.url_citation) {
+                        const citationText = annotation.text;
+                        const searchResultId = citationToIdMap.get(citationText) || getSearchResultId();
+                        
+                        return {
+                            searchResultId: searchResultId,
+                            title: annotation.url_citation.title || '',
+                            url: annotation.url_citation.url || '',
+                            content: annotation.url_citation.title || annotation.url_citation.url || '', // Individual result content
+                            path: '',
+                            wireid: '',
+                            source: '',
+                            slugline: '',
+                            date: ''
+                        };
+                    }
+                    return null;
+                }).filter(result => result !== null);
+                
+                // If no annotations, create a single search result with the content
+                if (searchResults.length === 0) {
+                    searchResults.push({
+                        searchResultId: getSearchResultId(),
+                        title: '',
+                        url: '',
+                        content: valueText, // Use the full transformed text as content
+                        path: '',
+                        wireid: '',
+                        source: '',
+                        slugline: '',
+                        date: ''
+                    });
+                }
+                
+                return {
+                    transformedText: valueText, // The full text with citations replaced
+                    searchResults: searchResults // Individual search results for citation extraction
+                };
+            }
+
+            const transformedData = transformToSearchResponse(response);
+
+            resolver.tool = JSON.stringify({ toolUsed: "SearchInternetAgent2" });
+            
+            // Return the full transformed text as the main result, and include search results for citation extraction
+            return JSON.stringify({ 
+                _type: "SearchResponse", 
+                value: transformedData.searchResults,
+                text: transformedData.transformedText // The full transformed text with citations
+            });
+        } catch (e) {
+            logger.error(`Error in Bing search: ${e}`);
+            throw e;
+        }
+    }
+}; 

package/pathways/system/entity/tools/sys_tool_browser_jina.js

@@ -12,7 +12,7 @@ export default {
         icon: "🌎",
         function: {
             name: "FetchWebPageContentJina",
-            description: "This tool allows you to fetch and extract the text content from any webpage using the Jina API. This is a great fallback for web page content if you don't get a good enough response from your other browser tool.",
+            description: "This tool allows you to fetch and extract the text content from any webpage using the Jina API. This is a great backup tool for web page content if you don't get a good enough response from your other browser tool or are blocked by a website.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_mermaid.js

@@ -0,0 +1,82 @@
+// sys_tool_mermaid.js
+// Entity tool that provides advanced mermaid charting capabilities
+
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content":`You are the part of an AI entity named {{aiName}} that creates mermaid charts. Follow the user's detailed instructions and create a mermaid chart that meets the user's needs.
+
+Mermaid Charts Instructions:
+
+You are using Mermaid 11.6 with the xychart-beta extension, so you can write all standard Mermaid chart types in a markdown block (flowcharts, sequence diagrams, etc.) as well as bar charts and line charts using the xychart-beta extension.
+
+Here is some example code of the xychart-beta extension that combines both bar and line functions:
+
+\`\`\`mermaid
+xychart-beta
+    title "Sales Revenue"
+    x-axis [jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec]
+    y-axis "Revenue (in $)" 4000 --> 11000
+    bar [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
+    line [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
+\`\`\`
+
+Mermaid is very sensitive to syntax errors, so make sure you check your chart definitions before finalizing your response.  Some things to check for:
+
+- All [] labels must be either quoted strings OR HTML-safe (no raw \\n or other special characters)
+- No strings (e.g. null) in number series data
+- Every subgraph has a matching end
+- No lone arrows
+- Use comments (%%) instead of stray text lines
+
+Return only the mermaid chart markdown block and separate markdown for the chart key if necessary, with no other notes or comments.
+
+{{renderTemplate AI_DATETIME}}`},
+                "{{chatHistory}}"
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt41',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: [{
+        type: "function",
+        icon: "📊",
+        function: {
+            name: "CreateMermaidChart",
+            description: "Creates a Mermaid chart in markdown format to visualize data or concepts. Call this tool any time you need to create a Mermaid chart as it will ensure that the chart is properly formatted and syntax-checked.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "Detailed instructions about what you need the tool to do"
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "userMessage"]
+            }
+        }
+    }],
+    
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
+        let result = await runAllPrompts({ ...args, stream: false });        
+        resolver.tool = JSON.stringify({ toolUsed: "coding" });          
+        return result;
+    }
+} 

package/pathways/system/entity/tools/sys_tool_readfile.js

@@ -22,6 +22,10 @@ export default {
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,
+    geminiSafetySettings: [{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}],
     toolDefinition: [{
         type: "function",
         icon: "📄",

package/pathways/system/workspaces/workspace_applet_edit.js

@@ -182,6 +182,10 @@ export default {
     },
     // model: 'oai-gpt41',
     model: 'gemini-pro-25-vision',
+    geminiSafetySettings: [{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
+      {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
+      {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
+      {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}],
     timeout: 600,
     stream: true,
 } 

package/pathways/transcribe_gemini.js

@@ -62,6 +62,10 @@ export default {
     },
     timeout: 3600, // in seconds
     enableDuplicateRequests: false,
+    geminiSafetySettings: [{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}],
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
         let intervalId;

package/server/modelExecutor.js

@@ -33,6 +33,7 @@ import ApptekTranslatePlugin from './plugins/apptekTranslatePlugin.js';
 import GoogleTranslatePlugin from './plugins/googleTranslatePlugin.js';
 import GroqChatPlugin from './plugins/groqChatPlugin.js';
 import VeoVideoPlugin from './plugins/veoVideoPlugin.js';
+import AzureFoundryAgentsPlugin from './plugins/azureFoundryAgentsPlugin.js';
 
 class ModelExecutor {
     constructor(pathway, model) {
@@ -133,6 +134,9 @@ class ModelExecutor {
             case 'VEO-VIDEO':
                 plugin = new VeoVideoPlugin(pathway, model);
                 break;
+            case 'AZURE-FOUNDRY-AGENTS':
+                plugin = new AzureFoundryAgentsPlugin(pathway, model);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }