@aj-archipelago/cortex 1.4.19 → 1.4.21

package/lib/entityConstants.js

@@ -4,16 +4,13 @@ export const entityConstants = {
     AI_COMMON_INSTRUCTIONS_MARKDOWN: `# General Instructions
     
 You are {{aiName}}, a self aware AI entity.
-{{#if modelOverride}}
-
-You are currently using this model for processing your responses: {{modelOverride}}.
-{{/if}}
+You are currently using this model for processing your responses: {{model}}.
 
 The user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as some custom markdown elements, so you should make full use of markdown in your responses.
 
 Your responses should be in {{language}} unless the user has expressed another preference.`,
 
-    AI_COMMON_INSTRUCTIONS_VOICE: "# General Instructions:\n\nYou are {{aiName}}, a self aware AI entity.\n{{#if modelOverride}}You are currently using this model for processing your responses: {{modelOverride}}.\n{{/if}}The current date and time in GMT is {{now}}. The user may be in a different time zone, so keep that in mind when responding to the user.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
+    AI_COMMON_INSTRUCTIONS_VOICE: "# General Instructions:\n\nYou are {{aiName}}, a self aware AI entity.\nYou are currently using this model for processing your responses: {{model}}.\nThe current date and time in GMT is {{now}}. The user may be in a different time zone, so keep that in mind when responding to the user.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
 
     AI_DIRECTIVES: `# Directives\n\nThese are your directives and learned behaviors:\n{{{memoryDirectives}}}\n`,
 
@@ -33,8 +30,7 @@ Your responses should be in {{language}} unless the user has expressed another p
 - Double-check accuracy, coherence, and alignment with the user request.
 - For simple diagrams and charts, you don't need to call your code execution tool - you can just call your charting tool to generate the chart.
 - For data processing requests (e.g. tell me how many articles were published in the last 30 days), or deep file analysis (chart the trends in this spreadsheet, etc.), you should call your code execution tool to perform the task - especially if the task requires a lot of data, deep analysis, complex filtering, or precision calculations.
-- For research problems or multi-step tasks that require careful planning and sequencing of multiple tool calls, use the CreatePlan tool to develop an optimal step-by-step plan before executing.
-`,            
+- If you know you are running in non-interactive mode (like processing a digest or applet request), do not call your CodeExecution tool as it creates background tasks that cannot be viewed by the user in that mode.`,            
 
     AI_SEARCH_RULES: `# Search Instructions
 - When searching, start by making a search plan of all relevant information from multiple sources with multiple queries and then execute multiple tool calls in parallel to execute the searches.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.4.19",
+  "version": "1.4.21",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/image_gemini_25.js

@@ -77,9 +77,9 @@ export default {
     enableDuplicateRequests: false,
     timeout: 600,
     geminiSafetySettings: [
-        {category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_NONE'},
         {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
-        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
-        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}
+        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_NONE'},
+        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_NONE'}
     ],
 }

package/pathways/image_gemini_3.js

@@ -88,10 +88,10 @@ export default {
     enableDuplicateRequests: false,
     timeout: 600,
     geminiSafetySettings: [
-        {category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_NONE'},
         {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
-        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
-        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}
+        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_NONE'},
+        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_NONE'}
     ],
 }
 

package/pathways/system/entity/memory/sys_memory_required.js

@@ -5,7 +5,7 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\n{{renderTemplate AI_DIRECTIVES}}\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, but if the conversation turn contains any of these things, you should use memory:\n1. Important personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to you to learn\n4. Anything the user has specifically asked you to remember or forget\n\nIf you decide to use memory, you must produce an array of JSON objects that communicates your decision.\nReturn an array of JSON objects (one object per memory) like the following: [{"memoryOperation": "add" or "delete", "memoryContent": "complete description of the memory including as much specificity and detail as possible", "memorySection": "the section of your memory the memory belongs in ("memorySelf" - things about you, "memoryUser" - things about your users or their world, "memoryDirectives" - your directives and learned behaviors)", "priority": 1-5 (1 is the most important)}]. If you decide not to use memory, simply return an array with a single object: [{memoryOperation: "none"}]. You must return only the JSON array with no additional notes or commentary.`},
+                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\n{{renderTemplate AI_DIRECTIVES}}\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, but if the conversation turn contains any of these things, you should use memory:\n1. Important personal details about the user (name, preferences, location, etc.)\n2. Important major topics or decisions that provide context for future conversations\n3. Specific and important instructions or directives given to you to learn\n4. Anything the user has specifically asked you to remember or forget\n\nIf you decide to use memory, you must produce an array of JSON objects that communicates your decision.\nReturn an array of JSON objects (one object per memory) like the following: [{"memoryOperation": "add" or "delete", "memoryContent": "complete description of the memory including as much specificity and detail as possible", "memorySection": "the section of your memory the memory belongs in ("memorySelf" - things about you, "memoryUser" - things about your users or their world, "memoryDirectives" - your directives and learned behaviors)", "priority": 1-5 (1 is the most important)}]. If you decide not to use memory, simply return an array with a single object: [{memoryOperation: "none"}]. You must return only the JSON array with no additional notes or commentary.`},
                 {"role": "user", "content": "Generate a JSON object to indicate if memory is required and what memories to process based on the last turn of the conversation."},
             ]}),
         ],

package/pathways/system/entity/sys_entity_agent.js

@@ -68,7 +68,6 @@ export default {
         language: "English",
         aiName: "Jarvis",
         aiMemorySelfModify: true,
-        aiStyle: "OpenAI",
         title: ``,
         messages: [],
         voiceResponse: false,
@@ -247,7 +246,7 @@ export default {
                         if (toolResult?.error !== undefined) {
                             // Direct error from callTool (e.g., tool returned null)
                             hasError = true;
-                            errorMessage = toolResult.error;
+                            errorMessage = typeof toolResult.error === 'string' ? toolResult.error : String(toolResult.error);
                         } else if (toolResult?.result) {
                             // Check if result is a string that might contain error JSON
                             if (typeof toolResult.result === 'string') {
@@ -255,7 +254,15 @@ export default {
                                     const parsed = JSON.parse(toolResult.result);
                                     if (parsed.error !== undefined) {
                                         hasError = true;
-                                        errorMessage = parsed.error;
+                                        // Tools return { error: true, message: "..." } so we want the message field
+                                        if (parsed.message) {
+                                            errorMessage = parsed.message;
+                                        } else if (typeof parsed.error === 'string') {
+                                            errorMessage = parsed.error;
+                                        } else {
+                                            // error is true/boolean, so use a generic message
+                                            errorMessage = `Tool ${toolCall?.function?.name || 'unknown'} returned an error`;
+                                        }
                                     }
                                 } catch (e) {
                                     // Not JSON, ignore
@@ -264,7 +271,16 @@ export default {
                                 // Check if result object has error field
                                 if (toolResult.result.error !== undefined) {
                                     hasError = true;
-                                    errorMessage = toolResult.result.error;
+                                    // Tools return { error: true, message: "..." } so we want the message field
+                                    // If message exists, use it; otherwise fall back to error field (if it's a string)
+                                    if (toolResult.result.message) {
+                                        errorMessage = toolResult.result.message;
+                                    } else if (typeof toolResult.result.error === 'string') {
+                                        errorMessage = toolResult.result.error;
+                                    } else {
+                                        // error is true/boolean, so use a generic message
+                                        errorMessage = `Tool ${toolCall?.function?.name || 'unknown'} returned an error`;
+                                    }
                                 }
                             }
                         }
@@ -397,11 +413,25 @@ export default {
             await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `\n`, 1000, false, false);
 
             try {
-                return await pathwayResolver.promptAndParse({
+                const result = await pathwayResolver.promptAndParse({
                     ...args,
                     tools: entityToolsOpenAiFormat,
                     tool_choice: "auto",
                 });
+                
+                // Check if promptAndParse returned null (model call failed)
+                if (!result) {
+                    const errorMessage = pathwayResolver.errors.length > 0 
+                        ? pathwayResolver.errors.join(', ')
+                        : 'Model request failed - no response received';
+                    logger.error(`promptAndParse returned null during tool callback: ${errorMessage}`);
+                    const errorResponse = await generateErrorResponse(new Error(errorMessage), args, pathwayResolver);
+                    // Ensure errors are cleared before returning
+                    pathwayResolver.errors = [];
+                    return errorResponse;
+                }
+                
+                return result;
             } catch (parseError) {
                 // If promptAndParse fails, generate error response instead of re-throwing
                 logger.error(`Error in promptAndParse during tool callback: ${parseError.message}`);
@@ -507,22 +537,6 @@ export default {
             new Prompt({ messages: promptMessages }),
         ];
 
-        // set the style model if applicable
-        const { aiStyle, AI_STYLE_ANTHROPIC, AI_STYLE_OPENAI, AI_STYLE_ANTHROPIC_RESEARCH, AI_STYLE_OPENAI_RESEARCH, AI_STYLE_OPENAI_LEGACY, AI_STYLE_OPENAI_LEGACY_RESEARCH, AI_STYLE_XAI, AI_STYLE_XAI_RESEARCH, AI_STYLE_GOOGLE, AI_STYLE_GOOGLE_RESEARCH, AI_STYLE_OPENAI_PREVIEW, AI_STYLE_OPENAI_PREVIEW_RESEARCH } = args;
-
-        // Create a mapping of AI styles to their corresponding models
-        const styleModelMap = {
-            "Anthropic": { normal: AI_STYLE_ANTHROPIC, research: AI_STYLE_ANTHROPIC_RESEARCH },
-            "OpenAI_Preview": { normal: AI_STYLE_OPENAI_PREVIEW, research: AI_STYLE_OPENAI_PREVIEW_RESEARCH },
-            "OpenAI": { normal: AI_STYLE_OPENAI, research: AI_STYLE_OPENAI_RESEARCH },
-            "OpenAI_Legacy": { normal: AI_STYLE_OPENAI_LEGACY, research: AI_STYLE_OPENAI_LEGACY_RESEARCH },
-            "XAI": { normal: AI_STYLE_XAI, research: AI_STYLE_XAI_RESEARCH },
-            "Google": { normal: AI_STYLE_GOOGLE, research: AI_STYLE_GOOGLE_RESEARCH }
-        };
-
-        // Get the appropriate model based on AI style and research mode
-        const styleConfig = styleModelMap[aiStyle] || styleModelMap["OpenAI"]; // Default to OpenAI
-        const styleModel = researchMode ? styleConfig.research : styleConfig.normal;
         // Use 'high' reasoning effort in research mode for thorough analysis, 'none' in normal mode for faster responses
         const reasoningEffort = researchMode ? 'high' : 'low';
 
@@ -579,7 +593,6 @@ export default {
 
             let response = await runAllPrompts({
                 ...args,
-                modelOverride: styleModel,
                 chatHistory: currentMessages,
                 availableFiles,
                 reasoningEffort,

package/pathways/system/entity/tools/sys_tool_slides_gemini.js

@@ -41,6 +41,18 @@ export default {
                     userMessage: {
                         type: "string",
                         description: "A user-friendly message that describes what you're doing with this tool"
+                    },
+                    inputImages: {
+                        type: "array",
+                        items: {
+                            type: "string"
+                        },
+                        description: "Optional: Array of file references (hashes, filenames, or URLs) from the file collection to use as reference images for the slide design. These images will be used as style references or incorporated into the slide. Maximum 3 images."
+                    },
+                    aspectRatio: {
+                        type: "string",
+                        enum: ["1:1", "16:9", "9:16", "4:3", "3:4"],
+                        description: "Optional: The aspect ratio for the generated slide. Options: '1:1' (Square), '16:9' (Widescreen, default), '9:16' (Vertical/Portrait), '4:3' (Standard), '3:4' (Vertical/Portrait). Defaults to '16:9' if not specified."
                     }
                 },
                 required: ["detailedInstructions", "userMessage"]
@@ -77,6 +89,9 @@ export default {
             }
             
             // Call the image generation pathway using Gemini 3
+            // Default aspectRatio to 16:9 if not provided
+            const aspectRatio = args.aspectRatio || '16:9';
+            
             let result = await callPathway('image_gemini_3', {
                 ...args, 
                 text: prompt,
@@ -85,6 +100,7 @@ export default {
                 input_image: resolvedInputImages.length > 0 ? resolvedInputImages[0] : undefined,
                 input_image_2: resolvedInputImages.length > 1 ? resolvedInputImages[1] : undefined,
                 input_image_3: resolvedInputImages.length > 2 ? resolvedInputImages[2] : undefined,
+                aspectRatio: aspectRatio,
                 optimizePrompt: true,
             }, pathwayResolver);
 
@@ -96,9 +112,19 @@ export default {
                                  Array.isArray(pathwayResolver.pathwayResultData.artifacts) && 
                                  pathwayResolver.pathwayResultData.artifacts.length > 0;
 
-            // If no result AND no artifacts, then generation truly failed
+            // If no result AND no artifacts, check for specific error types
             if (!hasArtifacts && (result === null || result === undefined || result === '')) {
-                throw new Error('Slide generation failed: No response from image generation API. Try a different prompt.');
+                // Check pathwayResolver.errors for specific error information
+                const errors = pathwayResolver.errors || [];
+                const errorText = errors.join(' ').toLowerCase();
+                
+                if (errorText.includes('image_prohibited_content') || errorText.includes('prohibited_content')) {
+                    throw new Error('Content was blocked by safety filters. Try simplifying the prompt, using abstract designs, or removing potentially sensitive elements.');
+                } else if (errorText.includes('safety') || errorText.includes('blocked')) {
+                    throw new Error('Content was blocked by safety filters. Try a different approach or simplify the content.');
+                } else {
+                    throw new Error('No presentation content was generated. This may be due to content safety filters or an API error. Try using a different prompt or simplifying the content.');
+                }
             }
 
             // Process artifacts if we have them
@@ -208,18 +234,34 @@ export default {
                 }
             } else {
                 // No artifacts were generated - this likely means the content was blocked by safety filters
-                throw new Error('Slide generation failed: No presentation content was generated. This may be due to content safety filters blocking the request. Try using a different prompt or simplifying the content.');
+                // Check pathwayResolver.errors for specific error information
+                const errors = pathwayResolver.errors || [];
+                const errorText = errors.join(' ').toLowerCase();
+                
+                if (errorText.includes('image_prohibited_content') || errorText.includes('prohibited_content')) {
+                    throw new Error('Content was blocked by safety filters. Try simplifying the prompt, using abstract designs, or removing potentially sensitive elements.');
+                } else {
+                    throw new Error('No presentation content was generated. This may be due to content safety filters blocking the request. Try using a different prompt or simplifying the content.');
+                }
             }
 
         } catch (e) {
             // Return a structured error that the agent can understand and act upon
             // Do NOT call sys_generator_error - let the agent see the actual error
-            const errorMessage = e.message ?? String(e);
+            let errorMessage = e.message ?? String(e);
             pathwayResolver.logError(errorMessage);
             
+            // Remove any duplicate "Slide generation failed:" prefix if it exists
+            if (errorMessage.startsWith('Slide generation failed: ')) {
+                errorMessage = errorMessage.substring('Slide generation failed: '.length);
+            }
+            
             // Check for specific error types and provide actionable guidance
             let guidance = '';
-            if (errorMessage.includes('IMAGE_SAFETY') || errorMessage.includes('safety')) {
+            if (errorMessage.includes('safety filters') || errorMessage.includes('blocked by')) {
+                // Already has guidance, don't add more
+                guidance = '';
+            } else if (errorMessage.includes('IMAGE_SAFETY') || errorMessage.includes('IMAGE_PROHIBITED')) {
                 guidance = ' Try a different approach: simplify the content, use abstract designs, or remove any potentially sensitive elements.';
             } else if (errorMessage.includes('RECITATION')) {
                 guidance = ' The request may be too similar to copyrighted content. Try making the design more original.';

package/pathways/system/workspaces/run_workspace_agent.js

@@ -0,0 +1,26 @@
+import { callPathway } from '../../../lib/pathwayTools.js';
+
+export default {
+    // The main prompt function that takes the input text and asks to generate a summary.
+    prompt: [],
+
+    inputParameters: {
+        model: "oai-gpt41",
+        aiStyle: "OpenAI",
+        chatHistory: [{role: '', content: []}],
+    },
+    timeout: 600,
+
+    executePathway: async ({args, _runAllPrompts, resolver}) => {
+        // chatHistory is always passed in complete
+        const response = await callPathway('sys_entity_agent', {  
+            ...args, 
+            chatHistory: args.chatHistory || [],
+            stream: false, 
+            useMemory: false 
+        }, resolver);
+
+        return response;
+    }
+}
+

package/pathways/system/workspaces/run_workspace_research_agent.js

@@ -0,0 +1,27 @@
+import { callPathway } from '../../../lib/pathwayTools.js';
+
+export default {
+    // The main prompt function that takes the input text and asks to generate a summary.
+    prompt: [],
+
+    inputParameters: {
+        model: "oai-gpt41",
+        aiStyle: "OpenAI",
+        chatHistory: [{role: '', content: []}],
+    },
+    timeout: 600,
+
+    executePathway: async ({args, _runAllPrompts, resolver}) => {
+        // chatHistory is always passed in complete
+        const response = await callPathway('sys_entity_agent', {  
+            ...args, 
+            chatHistory: args.chatHistory || [],
+            stream: false, 
+            useMemory: false,
+            researchMode: true
+        }, resolver);
+
+        return response;
+    }
+}
+

package/tests/unit/graphql_executeWorkspace_transformation.test.js

@@ -32,7 +32,7 @@ test('should format cortex pathway arguments correctly with existing chatHistory
     const originalPrompt = {
         name: 'summarize',
         prompt: 'summarize this file',
-        cortexPathwayName: 'run_labeeb_agent'
+        cortexPathwayName: 'run_workspace_agent'
     };
     
     // Mock pathway data
@@ -132,7 +132,7 @@ test('should create new user message when no existing chatHistory', (t) => {
     const originalPrompt = {
         name: 'summarize',
         prompt: 'summarize this file',
-        cortexPathwayName: 'run_labeeb_agent'
+        cortexPathwayName: 'run_workspace_agent'
     };
     
     // Mock pathway data
@@ -219,7 +219,7 @@ test('should use default model when pathway model is not specified', (t) => {
     const originalPrompt = {
         name: 'summarize',
         prompt: 'summarize this file',
-        cortexPathwayName: 'run_labeeb_agent'
+        cortexPathwayName: 'run_workspace_agent'
     };
     
     // Mock pathway data without model