@aj-archipelago/cortex 1.4.31 → 1.4.33

package/pathways/system/entity/sys_entity_agent.js

@@ -1,8 +1,11 @@
 // sys_entity_agent.js
 // Agentic extension of the entity system that uses OpenAI's tool calling API
 const MAX_TOOL_CALLS = 50;
+const TOOL_TIMEOUT_MS = 120000; // 2 minute timeout per tool call
+const MAX_TOOL_RESULT_LENGTH = 50000; // Truncate oversized tool results to prevent context overflow
 
-import { callPathway, callTool, say, sendToolStart, sendToolFinish } from '../../../lib/pathwayTools.js';
+import { callPathway, callTool, say, sendToolStart, sendToolFinish, withTimeout } from '../../../lib/pathwayTools.js';
+import { publishRequestProgress } from '../../../lib/redisSubscription.js';
 import logger from '../../../lib/logger.js';
 import { config } from '../../../config.js';
 import { syncAndStripFilesFromChatHistory } from '../../../lib/fileUtils.js';
@@ -79,7 +82,13 @@ export default {
         entityId: ``,
         researchMode: false,
         userInfo: '',
-        model: 'oai-gpt41'
+        model: 'oai-gpt41',
+        contextKey: ``,
+        clientSideTools: {
+            type: 'array',
+            items: { type: 'object' },
+            default: []
+        }
     },
     timeout: 600,
 
@@ -132,10 +141,13 @@ export default {
                         // Create an isolated copy of messages for this tool
                         const toolMessages = JSON.parse(JSON.stringify(preToolCallMessages));
                         
-                        // Get the tool definition to check for icon
+                        // Get the tool definition to check for icon and timeout
                         const toolDefinition = entityTools[toolFunction]?.definition;
                         const toolIcon = toolDefinition?.icon || '🛠️';
                         
+                        // Get timeout from tool definition or use default
+                        const toolTimeout = toolDefinition?.timeout || TOOL_TIMEOUT_MS;
+                        
                         // Get the user message for the tool
                         const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name}`;
                         
@@ -149,13 +161,19 @@ export default {
                             // Continue execution even if start message fails
                         }
 
-                        const toolResult = await callTool(toolFunction, {
-                            ...args,
-                            ...toolArgs,
-                            toolFunction,
-                            chatHistory: toolMessages,
-                            stream: false
-                        }, entityTools, pathwayResolver);
+                        // Wrap tool call with timeout to prevent hanging
+                        const toolResult = await withTimeout(
+                            callTool(toolFunction, {
+                                ...args,
+                                ...toolArgs,
+                                toolFunction,
+                                chatHistory: toolMessages,
+                                stream: false,
+                                useMemory: false  // Disable memory synthesis for tool calls
+                            }, entityTools, pathwayResolver),
+                            toolTimeout,
+                            `Tool ${toolCall.function.name} timed out after ${toolTimeout / 1000}s`
+                        );
 
                         // Tool calls and results need to be paired together in the message history
                         // Add the tool call to the isolated message history
@@ -305,7 +323,9 @@ export default {
                             messages: toolMessages
                         };
                     } catch (error) {
-                        logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
+                        // Detect if this is a timeout error for clearer logging
+                        const isTimeout = error.message?.includes('timed out');
+                        logger.error(`${isTimeout ? 'Timeout' : 'Error'} executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
                         
                         // Send tool finish message (error)
                         // Get requestId and toolCallId if not already defined (in case error occurred before they were set)
@@ -406,7 +426,17 @@ export default {
                 );
             }
 
-            args.chatHistory = finalMessages;
+            // Truncate oversized tool results to prevent context overflow
+            args.chatHistory = finalMessages.map(msg => {
+                if (msg.role === 'tool' && msg.content && msg.content.length > MAX_TOOL_RESULT_LENGTH) {
+                    logger.warn(`Truncating oversized tool result (${msg.content.length} chars) for ${msg.name || 'unknown tool'}`);
+                    return {
+                        ...msg,
+                        content: msg.content.substring(0, MAX_TOOL_RESULT_LENGTH) + '\n\n[Content truncated due to length]'
+                    };
+                }
+                return msg;
+            });
 
             // clear any accumulated pathwayResolver errors from the tools
             pathwayResolver.errors = [];
@@ -423,16 +453,27 @@ export default {
                 
                 // Check if promptAndParse returned null (model call failed)
                 if (!result) {
-                    const errorMessage = pathwayResolver.errors.length > 0 
+                    const errorMessage = pathwayResolver.errors.length > 0
                         ? pathwayResolver.errors.join(', ')
                         : 'Model request failed - no response received';
                     logger.error(`promptAndParse returned null during tool callback: ${errorMessage}`);
                     const errorResponse = await generateErrorResponse(new Error(errorMessage), args, pathwayResolver);
                     // Ensure errors are cleared before returning
                     pathwayResolver.errors = [];
+
+                    // In streaming mode, the toolCallback is invoked fire-and-forget by the plugin,
+                    // so we must stream the error response directly to the client and close the stream
+                    const requestId = pathwayResolver.rootRequestId || pathwayResolver.requestId;
+                    publishRequestProgress({
+                        requestId,
+                        progress: 1,
+                        data: JSON.stringify(errorResponse),
+                        info: JSON.stringify(pathwayResolver.pathwayResultData || {}),
+                        error: ''
+                    });
                     return errorResponse;
                 }
-                
+
                 return result;
             } catch (parseError) {
                 // If promptAndParse fails, generate error response instead of re-throwing
@@ -440,6 +481,17 @@ export default {
                 const errorResponse = await generateErrorResponse(parseError, args, pathwayResolver);
                 // Ensure errors are cleared before returning
                 pathwayResolver.errors = [];
+
+                // In streaming mode, the toolCallback is invoked fire-and-forget by the plugin,
+                // so we must stream the error response directly to the client and close the stream
+                const requestId = pathwayResolver.rootRequestId || pathwayResolver.requestId;
+                publishRequestProgress({
+                    requestId,
+                    progress: 1,
+                    data: JSON.stringify(errorResponse),
+                    info: JSON.stringify(pathwayResolver.pathwayResultData || {}),
+                    error: ''
+                });
                 return errorResponse;
             }
         }
@@ -449,16 +501,48 @@ export default {
         let pathwayResolver = resolver;
 
         // Load input parameters and information into args
-        const { entityId, voiceResponse, aiMemorySelfModify, chatId, researchMode } = { ...pathwayResolver.pathway.inputParameters, ...args };
+        let { entityId, voiceResponse, aiMemorySelfModify, chatId, researchMode, clientSideTools } = { ...pathwayResolver.pathway.inputParameters, ...args };
+        
+        // Parse clientSideTools if it's a string (from GraphQL)
+        if (typeof clientSideTools === 'string') {
+            try {
+                clientSideTools = JSON.parse(clientSideTools);
+            } catch (e) {
+                logger.error(`Failed to parse clientSideTools: ${e.message}`);
+                clientSideTools = [];
+            }
+        }
         
         const entityConfig = loadEntityConfig(entityId);
-        const { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
+        let { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
         const { name: entityName, instructions: entityInstructions } = entityConfig || {};
         
         // Determine useMemory: entityConfig.useMemory === false is a hard disable (entity can't use memory)
         // Otherwise args.useMemory can disable it, default true
         args.useMemory = entityConfig?.useMemory === false ? false : (args.useMemory ?? true);
 
+        // Add client-side tools from the caller
+        if (clientSideTools && Array.isArray(clientSideTools) && clientSideTools.length > 0) {
+            logger.info(`Adding ${clientSideTools.length} client-side tools from caller`);
+            clientSideTools.forEach(tool => {
+                const toolName = tool.function?.name?.toLowerCase();
+                if (toolName) {
+                    // Mark as client-side tool and add to available tools
+                    entityTools[toolName] = {
+                        definition: {
+                            ...tool,
+                            clientSide: true,  // Mark it as client-side
+                            icon: tool.icon || '📱'
+                        },
+                        pathwayName: 'client_side_execution',  // Placeholder pathway
+                        clientSide: true
+                    };
+                    entityToolsOpenAiFormat.push(tool);
+                    logger.info(`Registered client-side tool: ${toolName}`);
+                }
+            });
+        }
+
         // Initialize chat history if needed
         if (!args.chatHistory || args.chatHistory.length === 0) {
             args.chatHistory = [];
@@ -562,10 +646,10 @@ export default {
 
         // truncate the chat history in case there is really long content
         const truncatedChatHistory = resolver.modelExecutor.plugin.truncateMessagesToTargetLength(args.chatHistory, null, 1000);
-      
+
         // Asynchronously manage memory for this context
         if (args.aiMemorySelfModify && args.useMemory) {
-            callPathway('sys_memory_manager', {  ...args, chatHistory: truncatedChatHistory, stream: false })    
+            callPathway('sys_memory_manager', {  ...args, chatHistory: truncatedChatHistory, stream: false })
             .catch(error => logger.error(error?.message || "Error in sys_memory_manager pathway"));
         }
 
@@ -594,6 +678,10 @@ export default {
             memoryLookupRequired = false;
         }
 
+        // Update pathwayResolver.args with stripped chatHistory
+        // This ensures toolCallback receives the processed history, not the original
+        pathwayResolver.args = {...args};
+
         try {
             let currentMessages = JSON.parse(JSON.stringify(args.chatHistory));
 

package/pathways/transcribe_gemini.js

@@ -225,7 +225,7 @@ REMEMBER:
                     .sort((a, b) => a.index - b.index)
                     .map(item => item.result);
             } catch (error) {
-                logger.error('Error processing chunks:', error);
+                logger.error(`Error processing chunks: ${error.message}`);
                 throw error;
             }
         };

package/server/clientToolCallbacks.js

@@ -0,0 +1,241 @@
+// clientToolCallbacks.js
+// Storage and management for pending client-side tool callbacks
+
+import logger from '../lib/logger.js';
+import Redis from 'ioredis';
+import { config } from '../config.js';
+
+// Map to store pending client tool callbacks
+// Key: toolCallbackId, Value: { resolve, reject, timeout, requestId }
+const pendingCallbacks = new Map();
+
+// Default timeout for client tool responses (5 minutes)
+// Increased from 60s to 5min to accommodate longer operations like CreateApplet
+const DEFAULT_TIMEOUT = 300000;
+
+// Redis setup for cross-instance communication
+const connectionString = config.get('storageConnectionString');
+const clientToolCallbackChannel = 'clientToolCallbacks';
+
+let subscriptionClient;
+let publisherClient;
+
+if (connectionString) {
+    logger.info(`Setting up Redis pub/sub for client tool callbacks on channel: ${clientToolCallbackChannel}`);
+    
+    try {
+        subscriptionClient = new Redis(connectionString);
+        subscriptionClient.on('error', (error) => {
+            logger.error(`Redis subscriptionClient error (clientToolCallbacks): ${error}`);
+        });
+        
+        subscriptionClient.on('connect', () => {
+            subscriptionClient.subscribe(clientToolCallbackChannel, (error) => {
+                if (error) {
+                    logger.error(`Error subscribing to Redis channel ${clientToolCallbackChannel}: ${error}`);
+                } else {
+                    logger.info(`Subscribed to client tool callback channel: ${clientToolCallbackChannel}`);
+                }
+            });
+        });
+        
+        subscriptionClient.on('message', (channel, message) => {
+            if (channel === clientToolCallbackChannel) {
+                try {
+                    const { toolCallbackId, result } = JSON.parse(message);
+                    logger.debug(`Received client tool callback via Redis: ${toolCallbackId}`);
+                    
+                    // Try to resolve it locally (will only work if this instance has the pending callback)
+                    resolveClientToolCallbackLocal(toolCallbackId, result);
+                } catch (error) {
+                    logger.error(`Error processing client tool callback from Redis: ${error}`);
+                }
+            }
+        });
+    } catch (error) {
+        logger.error(`Redis connection error (clientToolCallbacks): ${error}`);
+    }
+    
+    try {
+        publisherClient = new Redis(connectionString);
+        publisherClient.on('error', (error) => {
+            logger.error(`Redis publisherClient error (clientToolCallbacks): ${error}`);
+        });
+    } catch (error) {
+        logger.error(`Redis connection error (clientToolCallbacks): ${error}`);
+    }
+} else {
+    logger.info('No Redis connection configured. Client tool callbacks will only work on single instance.');
+}
+
+/**
+ * Register a pending client tool callback
+ * @param {string} toolCallbackId - Unique ID for this tool call
+ * @param {string} requestId - The request ID for logging/tracking
+ * @param {number} timeoutMs - Timeout in milliseconds
+ * @returns {Promise} Promise that resolves when client submits the result
+ */
+export function waitForClientToolResult(toolCallbackId, requestId, timeoutMs = DEFAULT_TIMEOUT) {
+    return new Promise((resolve, reject) => {
+        // Set up timeout
+        const timeout = setTimeout(() => {
+            pendingCallbacks.delete(toolCallbackId);
+            logger.error(`Client tool callback timeout for ${toolCallbackId} (requestId: ${requestId})`);
+            reject(new Error(`Client tool execution timeout after ${timeoutMs}ms`));
+        }, timeoutMs);
+
+        // Store the callback
+        pendingCallbacks.set(toolCallbackId, {
+            resolve,
+            reject,
+            timeout,
+            requestId,
+            createdAt: Date.now()
+        });
+
+        logger.info(`Registered client tool callback: ${toolCallbackId} (requestId: ${requestId})`);
+    });
+}
+
+/**
+ * Resolve a pending client tool callback locally (internal use)
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {object} result - The result from the client
+ * @returns {boolean} True if callback was found and resolved
+ */
+function resolveClientToolCallbackLocal(toolCallbackId, result) {
+    const callback = pendingCallbacks.get(toolCallbackId);
+    
+    if (!callback) {
+        // This is normal in a multi-instance setup - the callback might be on another instance
+        logger.debug(`No pending callback found for toolCallbackId: ${toolCallbackId} (may be on another instance)`);
+        return false;
+    }
+
+    // Clear the timeout
+    clearTimeout(callback.timeout);
+    
+    // Remove from pending
+    pendingCallbacks.delete(toolCallbackId);
+    
+    logger.info(`Resolved client tool callback: ${toolCallbackId} (requestId: ${callback.requestId})`);
+    
+    // Resolve the promise
+    callback.resolve(result);
+    
+    return true;
+}
+
+/**
+ * Resolve a pending client tool callback with the result
+ * This function publishes to Redis so all instances can attempt to resolve
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {object} result - The result from the client
+ * @returns {Promise<boolean>} True if callback was published/resolved
+ */
+export async function resolveClientToolCallback(toolCallbackId, result) {
+    if (publisherClient) {
+        // Publish to Redis so all instances can try to resolve
+        try {
+            const message = JSON.stringify({ toolCallbackId, result });
+            logger.debug(`Publishing client tool callback to Redis: ${toolCallbackId}`);
+            await publisherClient.publish(clientToolCallbackChannel, message);
+            return true;
+        } catch (error) {
+            logger.error(`Error publishing client tool callback to Redis: ${error}`);
+            // Fall back to local resolution
+            return resolveClientToolCallbackLocal(toolCallbackId, result);
+        }
+    } else {
+        // No Redis, resolve locally
+        return resolveClientToolCallbackLocal(toolCallbackId, result);
+    }
+}
+
+/**
+ * Reject a pending client tool callback locally (internal use)
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {Error} error - The error
+ * @returns {boolean} True if callback was found and rejected
+ */
+function rejectClientToolCallbackLocal(toolCallbackId, error) {
+    const callback = pendingCallbacks.get(toolCallbackId);
+    
+    if (!callback) {
+        logger.debug(`No pending callback found for toolCallbackId: ${toolCallbackId} (may be on another instance)`);
+        return false;
+    }
+
+    // Clear the timeout
+    clearTimeout(callback.timeout);
+    
+    // Remove from pending
+    pendingCallbacks.delete(toolCallbackId);
+    
+    logger.info(`Rejected client tool callback: ${toolCallbackId} (requestId: ${callback.requestId})`);
+    
+    // Reject the promise
+    callback.reject(error);
+    
+    return true;
+}
+
+/**
+ * Reject a pending client tool callback with an error
+ * This function publishes to Redis so all instances can attempt to reject
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {Error} error - The error
+ * @returns {Promise<boolean>} True if callback was published/rejected
+ */
+export async function rejectClientToolCallback(toolCallbackId, error) {
+    if (publisherClient) {
+        // Publish to Redis so all instances can try to reject
+        try {
+            const message = JSON.stringify({ 
+                toolCallbackId, 
+                result: { success: false, error: error.message || error.toString() } 
+            });
+            logger.debug(`Publishing client tool callback rejection to Redis: ${toolCallbackId}`);
+            await publisherClient.publish(clientToolCallbackChannel, message);
+            return true;
+        } catch (publishError) {
+            logger.error(`Error publishing client tool callback rejection to Redis: ${publishError}`);
+            // Fall back to local rejection
+            return rejectClientToolCallbackLocal(toolCallbackId, error);
+        }
+    } else {
+        // No Redis, reject locally
+        return rejectClientToolCallbackLocal(toolCallbackId, error);
+    }
+}
+
+/**
+ * Get count of pending callbacks (for monitoring)
+ */
+export function getPendingCallbackCount() {
+    return pendingCallbacks.size;
+}
+
+/**
+ * Clean up old callbacks (for maintenance)
+ */
+export function cleanupOldCallbacks(maxAgeMs = 120000) {
+    const now = Date.now();
+    let cleaned = 0;
+    
+    for (const [id, callback] of pendingCallbacks.entries()) {
+        if (now - callback.createdAt > maxAgeMs) {
+            clearTimeout(callback.timeout);
+            pendingCallbacks.delete(id);
+            callback.reject(new Error('Callback expired during cleanup'));
+            cleaned++;
+        }
+    }
+    
+    if (cleaned > 0) {
+        logger.info(`Cleaned up ${cleaned} old client tool callbacks`);
+    }
+    
+    return cleaned;
+}
+

package/server/executeWorkspace.js

@@ -80,6 +80,7 @@ const resolveAndAddFileContent = async (pathways, pathwayArgs, requestId, config
 
 // Helper function to execute pathway with cortex pathway name or fallback to legacy
 const executePathwayWithFallback = async (pathway, pathwayArgs, contextValue, info, requestId, originalPrompt = null, config) => {
+    // Extract cortexPathwayName from originalPrompt (could be original object or transformed Prompt object)
     const cortexPathwayName = (originalPrompt && typeof originalPrompt === 'object' && originalPrompt.cortexPathwayName) 
         ? originalPrompt.cortexPathwayName 
         : null;
@@ -100,6 +101,12 @@ const executePathwayWithFallback = async (pathway, pathwayArgs, contextValue, in
             // Remove old aiStyle parameter (no longer used)
             delete cortexArgs.aiStyle;
             
+            // Extract researchMode from originalPrompt if not already in pathwayArgs
+            // originalPrompt could be the original object from JSON or a transformed Prompt object
+            if (originalPrompt && typeof originalPrompt === 'object' && originalPrompt.researchMode !== undefined) {
+                cortexArgs.researchMode = originalPrompt.researchMode;
+            }
+            
             // Transform context parameters to agentContext array format (only if agentContext not already provided)
             if (!cortexArgs.agentContext && (cortexArgs.contextId || cortexArgs.contextKey || cortexArgs.altContextId || cortexArgs.altContextKey)) {
                 const agentContext = [];

package/server/graphql.js

@@ -20,7 +20,7 @@ import logger from '../lib/logger.js';
 import { buildModelEndpoints } from '../lib/requestExecutor.js';
 import { startTestServer } from '../tests/helpers/server.js';
 import { requestState } from './requestState.js';
-import { cancelRequestResolver } from './resolver.js';
+import { cancelRequestResolver, submitClientToolResultResolver } from './resolver.js';
 import subscriptions from './subscriptions.js';
 import { getMessageTypeDefs } from './typeDef.js';
 import { buildRestEndpoints } from './rest.js';
@@ -79,6 +79,7 @@ const getTypedefs = (pathways, pathwayManager) => {
 
     type Mutation {
         cancelRequest(requestId: String!): Boolean
+        submitClientToolResult(requestId: String!, toolCallbackId: String!, result: String!, success: Boolean!): Boolean
     }
 
     ${getExecuteWorkspaceTypeDefs()}
@@ -140,6 +141,7 @@ const getResolvers = (config, pathways, pathwayManager) => {
         },
         Mutation: {
             'cancelRequest': cancelRequestResolver,
+            'submitClientToolResult': submitClientToolResultResolver,
             ...mutationResolvers,
             ...pathwayManagerResolvers.Mutation
         },

package/server/modelExecutor.js

@@ -26,6 +26,7 @@ import Gemini3ImagePlugin from './plugins/gemini3ImagePlugin.js';
 import Gemini3ReasoningVisionPlugin from './plugins/gemini3ReasoningVisionPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
 import Claude4VertexPlugin from './plugins/claude4VertexPlugin.js';
+import ClaudeAnthropicPlugin from './plugins/claudeAnthropicPlugin.js';
 import NeuralSpacePlugin from './plugins/neuralSpacePlugin.js';
 import RunwareAiPlugin from './plugins/runwareAiPlugin.js';
 import ReplicateApiPlugin from './plugins/replicateApiPlugin.js';
@@ -122,6 +123,9 @@ class ModelExecutor {
             case 'CLAUDE-4-VERTEX':
                 plugin = new Claude4VertexPlugin(pathway, model);
                 break;
+            case 'CLAUDE-ANTHROPIC':
+                plugin = new ClaudeAnthropicPlugin(pathway, model);
+                break;
             case 'RUNWARE-AI':
                 plugin = new RunwareAiPlugin(pathway, model);
                 break;