@aj-archipelago/cortex 1.4.0 → 1.4.2

package/pathways/system/entity/tools/sys_tool_coding.js

@@ -23,9 +23,9 @@ export default {
     enableDuplicateRequests: false,
     timeout: 600,
     // Tool disabled for now
-    /*
     toolDefinition: [{
         type: "function",
+        enabled: false,
         icon: "💻",
         function: {
             name: "Code",
@@ -46,7 +46,6 @@ export default {
             }
         }
     }],
-    */
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
         if (args.detailedInstructions) {

package/pathways/system/entity/tools/sys_tool_grok_x_search.js

@@ -21,7 +21,7 @@ export default {
         icon: "🔍",
         function: {
             name: "SearchXPlatform",
-            description: "This tool allows you to search the X platform (formerly Twitter) for current posts, discussions, and real-time information. Use this for finding recent social media content, trending topics, public opinions, and real-time updates. Always call this tool in parallel rather than serially if you have several searches to do as it will be faster.",
+            description: "This tool allows you to search the X platform (formerly Twitter) for current posts, discussions, and real-time information. Use this for finding recent social media content, trending topics, public opinions, and real-time updates. This tool can be slow - 10-60s per search, so only use it when you really want X platform information. Always call this tool in parallel rather than serially if you have several searches to do as it will be faster.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_image.js

@@ -12,6 +12,7 @@ export default {
     timeout: 300,
     toolDefinition: [{
         type: "function",
+        enabled: false,
         icon: "🎨",
         function: {
             name: "GenerateImage",
@@ -37,7 +38,7 @@ export default {
         icon: "🔄",
         function: {
             name: "ModifyImage",
-            description: "Use when asked to modify, transform, or edit an existing image. This tool can apply various transformations like style changes, artistic effects, or specific modifications to an image that has been previously uploaded or generated. It takes up to two input images as a reference and outputs a new image based on the instructions. This tool does not display the image to the user - you need to do that with markdown in your response.",
+            description: "Use when asked to modify, transform, or edit an existing image. This tool can apply various transformations like style changes, artistic effects, or specific modifications to an image that has been previously uploaded or generated. It takes up to three input images as a reference and outputs a new image based on the instructions. This tool does not display the image to the user - you need to do that with markdown in your response.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_image_gemini.js

@@ -11,9 +11,9 @@ export default {
         model: 'oai-gpt4o',
     },
     timeout: 300,
-    /*
     toolDefinition: [{
         type: "function",
+        enabled: true,
         icon: "🎨",
         function: {
             name: "GenerateImage",
@@ -30,12 +30,13 @@ export default {
                         description: "A user-friendly message that describes what you're doing with this tool"
                     }
                 },
-                required: ["detailedInstructions", "renderText", "userMessage"]
+                required: ["detailedInstructions", "userMessage"]
             }
         }
     },
     {
         type: "function",
+        enabled: false,
         icon: "🔄",
         function: {
             name: "ModifyImage",
@@ -68,7 +69,6 @@ export default {
             }
         }
     }],
-    */
     executePathway: async ({args, runAllPrompts, resolver}) => {
         const pathwayResolver = resolver;
 

package/pathways/system/entity/tools/sys_tool_mermaid.js

@@ -2,49 +2,76 @@
 // Entity tool that provides advanced mermaid charting capabilities
 
 import { Prompt } from '../../../../server/prompt.js';
+import { validateMermaid, isValidMermaid, getDiagramType } from '@aj-archipelago/merval';
+import logger from '../../../../lib/logger.js';
 
-export default {
-    prompt:
-        [
-            new Prompt({ messages: [
-                {"role": "system", "content":`You are the part of an AI entity named {{aiName}} that creates mermaid charts. Follow the user's detailed instructions and create a mermaid chart that meets the user's needs.
-
-Mermaid Charts Instructions:
-
-You are using Mermaid 11.6 with the xychart-beta extension, so you can write all standard Mermaid chart types in a markdown block (flowcharts, sequence diagrams, etc.) as well as bar charts and line charts using the xychart-beta extension.
-
-Here is some example code of the xychart-beta extension that combines both bar and line functions:
-
-\`\`\`mermaid
-xychart-beta
-    title "Sales Revenue"
-    x-axis [jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec]
-    y-axis "Revenue (in $)" 4000 --> 11000
-    bar [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
-    line [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
-\`\`\`
-
-Mermaid is very sensitive to syntax errors, so make sure you check your chart definitions before finalizing your response.  Some things to check for:
+// Function to validate mermaid syntax using our lightweight validator
+function validateMermaidSyntax(mermaidCode) {
+    try {
+        // Ensure mermaidCode is a string
+        const codeStr = typeof mermaidCode === 'string' ? mermaidCode : String(mermaidCode);
+        
+        // Extract mermaid code from markdown block if present
+        const mermaidMatch = codeStr.match(/```mermaid\s*([\s\S]*?)\s*```/);
+        const codeToValidate = mermaidMatch ? mermaidMatch[1].trim() : codeStr.trim();
+        
+        if (!codeToValidate) {
+            return { isValid: false, error: "Empty mermaid code", diagramType: 'unknown' };
+        }
+        
+        // Use our lightweight validator
+        const result = validateMermaid(codeToValidate);
+        
+        return {
+            isValid: result.isValid,
+            error: result.isValid ? null : result.errors,
+            diagramType: result.diagramType,
+            ast: result.ast
+        };
+    } catch (error) {
+        return { 
+            isValid: false, 
+            error: `Validation error: ${error.message}`, 
+            diagramType: 'unknown' 
+        };
+    }
+}
 
-- All [] labels must be either quoted strings OR HTML-safe (no raw \\n or other special characters)
-- No strings (e.g. null) in number series data
-- Every subgraph has a matching end
-- No lone arrows
-- Use comments (%%) instead of stray text lines
+// Function to extract mermaid code from response
+function extractMermaidFromResponse(response) {
+    // Ensure response is a string
+    const responseStr = typeof response === 'string' ? response : String(response);
+    const mermaidMatch = responseStr.match(/```mermaid\s*([\s\S]*?)\s*```/);
+    return mermaidMatch ? mermaidMatch[1].trim() : null;
+}
 
-Return only the mermaid chart markdown block and separate markdown for the chart key if necessary, with no other notes or comments.
+// Function to format validation errors for detailed feedback
+function formatValidationErrors(errors) {
+    if (!errors || !Array.isArray(errors)) {
+        return 'Unknown validation error';
+    }
+    
+    return errors.map(error => {
+        let errorText = `Line ${error.line}, Column ${error.column}: ${error.message}`;
+        if (error.code) {
+            errorText += ` (Error Code: ${error.code})`;
+        }
+        if (error.suggestion) {
+            errorText += `\nSuggestion: ${error.suggestion}`;
+        }
+        return errorText;
+    }).join('\n\n');
+}
 
-{{renderTemplate AI_DATETIME}}`},
-                "{{chatHistory}}"
-            ]}),
-        ],
+export default {
+    prompt: [], // Prompts are set dynamically in executePathway
     inputParameters: {
         chatHistory: [{role: '', content: []}],
         contextId: ``,
         aiName: "Jarvis",
         language: "English",
     },
-    model: 'oai-gpt41',
+    model: 'oai-gpt5-chat',
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,
@@ -52,8 +79,8 @@ Return only the mermaid chart markdown block and separate markdown for the chart
         type: "function",
         icon: "📊",
         function: {
-            name: "CreateMermaidChart",
-            description: "Creates a Mermaid chart in markdown format to visualize data or concepts. Call this tool any time you need to create a Mermaid chart as it will ensure that the chart is properly formatted and syntax-checked.",
+            name: "CreateChart",
+            description: "Creates a single chart or diagram that will render in the UI to visualize data or concepts. You can create all the standard Mermaid chart types (flowcharts, sequence diagrams, gantt charts, etc.) as well as bar charts and line and scatter plots. This tool also validates the syntax and ensures proper formatting. Call this tool any time you need to create a chart outside of your coding agent. If you need to create multiple charts, you can call this tool multiple times in parallel to create multiple charts.",
             parameters: {
                 type: "object",
                 properties: {
@@ -75,8 +102,130 @@ Return only the mermaid chart markdown block and separate markdown for the chart
         if (args.detailedInstructions) {
             args.chatHistory.push({role: "user", content: args.detailedInstructions});
         }
-        let result = await runAllPrompts({ ...args, stream: false });        
-        resolver.tool = JSON.stringify({ toolUsed: "coding" });          
-        return result;
+        
+        const maxRetries = 10;
+        let attempts = 0;
+        let lastError = null;
+        let lastMermaidCode = null;
+        let pathwayResolver = resolver;
+        
+        while (attempts < maxRetries) {
+            attempts++;
+            
+            try {
+                let result;
+                
+                if (attempts === 1) {
+                    // First attempt: use full chat history for context
+                    // Set the initial prompt with full chat history
+                    pathwayResolver.pathwayPrompt = [
+                        new Prompt({ messages: [
+                            {"role": "system", "content":`You are the part of an AI entity named {{aiName}} that creates mermaid charts. Follow the user's detailed instructions and create a mermaid chart that meets the user's needs.
+
+Mermaid Charts Instructions:
+
+You are using Mermaid 11.6 with the xychart-beta extension, so you can write all standard Mermaid chart types in a markdown block (flowcharts, sequence diagrams, etc.) as well as bar charts and line charts using the xychart-beta extension.
+
+Here is some example code of the xychart-beta extension that combines both bar and line functions:
+
+\`\`\`mermaid
+xychart-beta
+    title "Sales Revenue"
+    x-axis [jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec]
+    y-axis "Revenue (in $)" 4000 --> 11000
+    bar [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
+    line [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
+\`\`\`
+
+Mermaid is very sensitive to syntax errors, so carefully consider your syntax before producing mermaid code.
+
+- All [] labels must be either quoted strings OR otherwise HTML-safe (no raw \\n or other special characters - this is the most common parsing issue - wonky labels)
+- No strings (e.g. null) in number series data
+- Every subgraph has a matching end
+- No lone arrows
+- Use comments (%%) instead of stray text lines for any text that is not part of the mermaid code
+
+Return only the mermaid chart markdown block with no other notes or comments.
+
+{{renderTemplate AI_DATETIME}}`},
+                            "{{chatHistory}}"
+                        ]})
+                    ];
+                    
+                    result = await runAllPrompts({ ...args, stream: false });
+                } else {
+                    // Retry attempts: use streamlined prompt with just the error and code
+                    pathwayResolver.pathwayPrompt = [
+                        new Prompt({ messages: [
+                            {"role": "system", "content":`You are fixing a mermaid chart syntax error. The previous attempt generated invalid mermaid code. Please fix the syntax errors and regenerate the chart.
+
+The error details below include line numbers, column positions, error codes, and suggestions. Use this information to precisely locate and fix the syntax issues.
+
+Focus only on fixing the syntax issues mentioned in the error details. Return only the corrected mermaid chart in markdown block format with no other comments.
+
+{{renderTemplate AI_DATETIME}}`},
+                            {"role": "user", "content": `Here is the mermaid code that was generated:\n\n\`\`\`mermaid\n${lastMermaidCode || ''}\n\`\`\`\n\nAnd here are the detailed error messages:\n\n${lastError || 'Unknown error'}\n\nPlease fix the syntax errors and regenerate the chart.`}
+                        ]})
+                    ];
+                    
+                    result = await runAllPrompts({ ...args, stream: false });
+                }
+                
+                // Extract mermaid code from the response
+                const mermaidCode = extractMermaidFromResponse(result);
+                
+                if (mermaidCode) {
+                    // Store the mermaid code for potential retry
+                    lastMermaidCode = mermaidCode;
+                    
+                    // Validate the mermaid chart using our lightweight validator
+                    const validation = validateMermaidSyntax(mermaidCode);
+                    
+                    if (validation.isValid) {
+                        pathwayResolver.tool = JSON.stringify({ 
+                            toolUsed: "CreateMermaidChart", 
+                            diagramType: validation.diagramType,
+                            attempts: attempts,
+                            validationPassed: true
+                        });
+                        
+                        // Return the validated mermaid chart
+                        return result;
+                    } else {
+                        const formattedErrors = formatValidationErrors(validation.error);
+                        logger.warn(`Mermaid chart has syntax errors: ${formattedErrors}`);
+                        lastError = formattedErrors;
+                        
+                        if (attempts < maxRetries) {
+                            continue; // Retry with streamlined prompt
+                        }
+                    }
+                } else {
+                    // No mermaid code found in response
+                    lastError = "No mermaid chart found in response";
+                    
+                    if (attempts < maxRetries) {
+                        // For retry, we'll use the streamlined prompt with the error message
+                        continue;
+                    }
+                }
+            } catch (error) {
+                lastError = error.message;
+                if (attempts < maxRetries) {
+                    continue; // Retry with streamlined prompt
+                }
+            }
+        }
+        
+        // If we've exhausted all retries, return the last result with error info
+        pathwayResolver.tool = JSON.stringify({ 
+            toolUsed: "CreateMermaidChart", 
+            error: lastError,
+            attempts: attempts,
+            validationFailed: true
+        });
+        
+        // Return a fallback response
+        return `Failed to generate valid mermaid chart after ${maxRetries} attempts. Last error: ${lastError}`;
     }
 } 

package/pathways/system/entity/tools/sys_tool_reasoning.js

@@ -24,6 +24,7 @@ export default {
     timeout: 600,
     toolDefinition: [{
         type: "function",
+        enabled: false,
         icon: "🗺️",
         function: {
             name: "PlanMultiStepTask",
@@ -46,6 +47,7 @@ export default {
     },
     {
         type: "function",
+        enabled: false,
         icon: "🧠",
         function: {
             name: "ApplyAdvancedReasoning",

package/pathways/system/entity/tools/sys_tool_verify.js

@@ -25,6 +25,7 @@ export default {
     timeout: 600,
     toolDefinition: [{
         type: "function",
+        enabled: false,
         icon: "🔍",
         function: {
             name: "VerifyResponse",
@@ -54,7 +55,6 @@ export default {
             }
         }
     }],
-    
     executePathway: async ({args, runAllPrompts, resolver}) => {        
         let result = await runAllPrompts({ ...args, stream: false });        
         resolver.tool = JSON.stringify({ toolUsed: "verification" });          

package/pathways/transcribe_gemini.js

@@ -273,12 +273,13 @@ REMEMBER:
         // }
         
         const result = await processChunksParallel(chunks, args);
+        const transcriptArray = result.map(item => item?.output_text || item);
         
         if (['srt','vtt'].includes(responseFormat.toLowerCase()) || wordTimestamped) { // align subtitles for formats
             const offsets = chunks.map((chunk, index) => chunk?.offset || index * OFFSET_CHUNK);
-            return alignSubtitles(result, responseFormat, offsets);
+            return alignSubtitles(transcriptArray, responseFormat, offsets);
         }
-        return result.join(` `);
+        return transcriptArray.join(` `);
     }catch(error){
         logger.error(`Error in transcribing: ${error}`);
         throw error;

package/server/graphql.js

@@ -181,7 +181,7 @@ const build = async (config) => {
 
     const server = new ApolloServer({
         schema: schema,
-        introspection: config.get('env') === 'development',
+        introspection: config.get('env') === 'development' || config.get('env') === 'debug',
         csrfPrevention: true,
         plugins: plugins.concat([// Proper shutdown for the HTTP server.
             ApolloServerPluginDrainHttpServer({ httpServer }),

package/server/pathwayResolver.js

@@ -6,6 +6,7 @@ import { getFirstNToken, getLastNToken, getSemanticChunks } from './chunker.js';
 import { PathwayResponseParser } from './pathwayResponseParser.js';
 import { Prompt } from './prompt.js';
 import { getv, setv } from '../lib/keyValueStorageClient.js';
+import { getvWithDoubleDecryption, setvWithDoubleEncryption } from '../lib/keyValueStorageClient.js';
 import { requestState } from './requestState.js';
 import { callPathway, addCitationsToResolver } from '../lib/pathwayTools.js';
 import logger from '../lib/logger.js';
@@ -392,12 +393,12 @@ class PathwayResolver {
             try {
                 // Load saved context and core memory if it exists
                 const [savedContext, memorySelf, memoryDirectives, memoryTopics, memoryUser, memoryContext] = await Promise.all([
-                    (getv && await getv(this.savedContextId)) || {},
-                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memorySelf', priority: 1, stripMetadata: true }),
-                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryDirectives', priority: 1, stripMetadata: true }),
-                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryTopics', priority: 0, numResults: 10 }),
-                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryUser', priority: 1, stripMetadata: true }),
-                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryContext', priority: 0 }),
+                    (getvWithDoubleDecryption && await getvWithDoubleDecryption(this.savedContextId, this.args?.contextKey)) || {},
+                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memorySelf', priority: 1, stripMetadata: true, contextKey: this.args?.contextKey }),
+                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryDirectives', priority: 1, stripMetadata: true, contextKey: this.args?.contextKey }),
+                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryTopics', priority: 0, numResults: 10, contextKey: this.args?.contextKey }),
+                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryUser', priority: 1, stripMetadata: true, contextKey: this.args?.contextKey }),
+                    callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryContext', priority: 0, contextKey: this.args?.contextKey }),
                 ]).catch(error => {
                     this.logError(`Failed to load memory: ${error.message}`);
                     return [{},'','','','',''];
@@ -435,7 +436,7 @@ class PathwayResolver {
             };
 
             if (currentState.savedContext !== this.initialState.savedContext) {
-                setv && await setv(this.savedContextId, this.savedContext);
+                setvWithDoubleEncryption && await setvWithDoubleEncryption(this.savedContextId, this.savedContext, this.args?.contextKey);
             }
         };
 

package/server/plugins/veoVideoPlugin.js

@@ -20,7 +20,9 @@ class VeoVideoPlugin extends ModelPlugin {
     const availableModels = {
       'veo-2.0-generate': 'GA',
       'veo-3.0-generate': 'Preview',
-      'veo-3.0-fast-generate': 'Preview'
+      'veo-3.0-fast-generate': 'Preview',
+      'veo-3.1-generate': 'Preview',
+      'veo-3.1-fast-generate': 'Preview'
     };
 
     // Get the model ID from the pathway or use default
@@ -53,6 +55,8 @@ class VeoVideoPlugin extends ModelPlugin {
         // generateAudio is required for 3.0 and not supported by 2.0
         ...(model === 'veo-3.0-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
         ...(model === 'veo-3.0-fast-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
+        ...(model === 'veo-3.1-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
+        ...(model === 'veo-3.1-fast-generate' && { generateAudio: combinedParameters.generateAudio !== undefined ? combinedParameters.generateAudio : true }),
         ...(combinedParameters.negativePrompt && { negativePrompt: combinedParameters.negativePrompt }),
         ...(combinedParameters.personGeneration && { personGeneration: combinedParameters.personGeneration }),
         ...(combinedParameters.sampleCount && { sampleCount: combinedParameters.sampleCount }),
@@ -74,6 +78,12 @@ class VeoVideoPlugin extends ModelPlugin {
       if (model === 'veo-3.0-fast-generate' && parameters.durationSeconds !== 8) {
         throw new Error(`Veo 3.0 only supports durationSeconds: 8, got: ${parameters.durationSeconds}`);
       }
+      if (model === 'veo-3.1-generate' && parameters.durationSeconds !== 8) {
+        throw new Error(`Veo 3.1 only supports durationSeconds: 8, got: ${parameters.durationSeconds}`);
+      }
+      if (model === 'veo-3.1-fast-generate' && parameters.durationSeconds !== 8) {
+        throw new Error(`Veo 3.1 only supports durationSeconds: 8, got: ${parameters.durationSeconds}`);
+      }
       if (model === 'veo-2.0-generate' && (parameters.durationSeconds < 5 || parameters.durationSeconds > 8)) {
         throw new Error(`Veo 2.0 supports durationSeconds between 5-8, got: ${parameters.durationSeconds}`);
       }
@@ -93,6 +103,18 @@ class VeoVideoPlugin extends ModelPlugin {
       if (model === 'veo-3.0-fast-generate' && parameters.video) {
         throw new Error('video parameter is not supported in Veo 3.0');
       }
+      if (model === 'veo-3.1-generate' && parameters.lastFrame) {
+        throw new Error('lastFrame parameter is not supported in Veo 3.1');
+      }
+      if (model === 'veo-3.1-generate' && parameters.video) {
+        throw new Error('video parameter is not supported in Veo 3.1');
+      }
+      if (model === 'veo-3.1-fast-generate' && parameters.lastFrame) {
+        throw new Error('lastFrame parameter is not supported in Veo 3.1');
+      }
+      if (model === 'veo-3.1-fast-generate' && parameters.video) {
+        throw new Error('video parameter is not supported in Veo 3.1');
+      }
     }
 
     // generateAudio constraints
@@ -105,6 +127,12 @@ class VeoVideoPlugin extends ModelPlugin {
     if (model === 'veo-3.0-fast-generate' && parameters.generateAudio === undefined) {
       logger.warn('generateAudio is required for Veo 3.0, defaulting to true');
     }
+    if (model === 'veo-3.1-generate' && parameters.generateAudio === undefined) {
+      logger.warn('generateAudio is required for Veo 3.1, defaulting to true');
+    }
+    if (model === 'veo-3.1-fast-generate' && parameters.generateAudio === undefined) {
+      logger.warn('generateAudio is required for Veo 3.1, defaulting to true');
+    }
   }
 
   // Execute the request to the Veo API