@aj-archipelago/cortex 1.4.31 → 1.4.32

package/lib/pathwayManager.js

@@ -349,6 +349,7 @@ class PathwayManager {
     const promptName = typeof promptItem === 'string' ? defaultName : (promptItem.name || defaultName);
     const promptFiles = typeof promptItem === 'string' ? [] : (promptItem.files || []);
     const cortexPathwayName = typeof promptItem === 'string' ? null : (promptItem.cortexPathwayName || null);
+    const researchMode = typeof promptItem === 'string' ? undefined : (promptItem.researchMode !== undefined ? promptItem.researchMode : undefined);
     
     const messages = [];
 
@@ -383,6 +384,11 @@ class PathwayManager {
       prompt.cortexPathwayName = cortexPathwayName;
     }
 
+    // Preserve researchMode if present
+    if (researchMode !== undefined) {
+      prompt.researchMode = researchMode;
+    }
+
     return prompt;
   }
 
@@ -460,6 +466,7 @@ class PathwayManager {
       prompt: String!
       files: [String!]
       cortexPathwayName: String
+      researchMode: Boolean
     }
     
     input PathwayInput {

package/lib/pathwayTools.js

@@ -6,6 +6,7 @@ import { getSemanticChunks } from "../server/chunker.js";
 import logger from '../lib/logger.js';
 import { requestState } from '../server/requestState.js';
 import { processPathwayParameters } from '../server/typeDef.js';
+import { waitForClientToolResult } from '../server/clientToolCallbacks.js';
 
 // callPathway - call a pathway from another pathway
 const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
@@ -91,6 +92,76 @@ const callTool = async (toolName, args, toolDefinitions, pathwayResolver) => {
     logger.debug(`callTool: Starting execution of ${toolName} ${JSON.stringify(logArgs)}`);
 
     try {
+        // Check if this is a client-side tool
+        if (toolDef.clientSide === true || toolDef.definition?.clientSide === true) {
+            logger.info(`Tool ${toolName} is a client-side tool - waiting for client execution`);
+            
+            const toolCallbackId = `${toolName}_${Date.now()}_${Math.random().toString(36).substring(7)}`;
+            
+            // Explicitly publish the marker to the stream so the client receives it
+            if (pathwayResolver) {
+                const requestId = pathwayResolver.rootRequestId || pathwayResolver.requestId;
+                
+                const toolCallbackData = {
+                    toolUsed: [toolName],
+                    clientSideTool: true,
+                    toolCallbackName: toolName,
+                    toolCallbackId: toolCallbackId,
+                    toolCallbackMessage: args.userMessage || `Executing ${toolName}...`,
+                    chatId: args.chatId || "",
+                    requestId: requestId, // Include requestId so client can submit tool results
+                    toolArgs: args
+                };
+                
+                try {
+                    logger.info(`Publishing client-side tool marker to requestId: ${requestId}, toolCallbackId: ${toolCallbackId}`);
+                    await publishRequestProgress({
+                        requestId,
+                        progress: 0.5,
+                        data: JSON.stringify(""),
+                        info: JSON.stringify(toolCallbackData)
+                    });
+                } catch (error) {
+                    logger.error(`Error publishing client-side tool marker: ${error.message}`);
+                    throw error;
+                }
+                
+                // Wait for the client to execute the tool and send back the result
+                logger.info(`Waiting for client tool result: ${toolCallbackId}`);
+                try {
+                    // Use 5 minute timeout to accommodate longer operations like CreateApplet
+                    const clientResult = await waitForClientToolResult(toolCallbackId, requestId, 300000);
+                    logger.info(`Received client tool result for ${toolCallbackId}: ${JSON.stringify(clientResult).substring(0, 200)}`);
+                    
+                    // If the client reported an error, throw it
+                    if (!clientResult.success) {
+                        throw new Error(clientResult.error || 'Client tool execution failed');
+                    }
+                    
+                    // Return the client's result
+                    toolResult = typeof clientResult.data === 'string' 
+                        ? clientResult.data 
+                        : JSON.stringify(clientResult.data);
+                    
+                    // Update resolver with tool result
+                    pathwayResolver.tool = JSON.stringify({
+                        ...toolCallbackData,
+                        result: clientResult.data
+                    });
+                    
+                    return {
+                        result: toolResult,
+                        images: []
+                    };
+                } catch (error) {
+                    logger.error(`Error waiting for client tool result: ${error.message}`);
+                    throw new Error(`Client tool execution failed: ${error.message}`);
+                }
+            } else {
+                throw new Error('PathwayResolver is required for client-side tools');
+            }
+        }
+
         const pathwayName = toolDef.pathwayName;
         // Merge hard-coded pathway parameters with runtime args
         const mergedArgs = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.4.31",
+  "version": "1.4.32",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/sys_entity_agent.js

@@ -79,7 +79,13 @@ export default {
         entityId: ``,
         researchMode: false,
         userInfo: '',
-        model: 'oai-gpt41'
+        model: 'oai-gpt41',
+        contextKey: ``,
+        clientSideTools: {
+            type: 'array',
+            items: { type: 'object' },
+            default: []
+        }
     },
     timeout: 600,
 
@@ -449,16 +455,48 @@ export default {
         let pathwayResolver = resolver;
 
         // Load input parameters and information into args
-        const { entityId, voiceResponse, aiMemorySelfModify, chatId, researchMode } = { ...pathwayResolver.pathway.inputParameters, ...args };
+        let { entityId, voiceResponse, aiMemorySelfModify, chatId, researchMode, clientSideTools } = { ...pathwayResolver.pathway.inputParameters, ...args };
+        
+        // Parse clientSideTools if it's a string (from GraphQL)
+        if (typeof clientSideTools === 'string') {
+            try {
+                clientSideTools = JSON.parse(clientSideTools);
+            } catch (e) {
+                logger.error(`Failed to parse clientSideTools: ${e.message}`);
+                clientSideTools = [];
+            }
+        }
         
         const entityConfig = loadEntityConfig(entityId);
-        const { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
+        let { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
         const { name: entityName, instructions: entityInstructions } = entityConfig || {};
         
         // Determine useMemory: entityConfig.useMemory === false is a hard disable (entity can't use memory)
         // Otherwise args.useMemory can disable it, default true
         args.useMemory = entityConfig?.useMemory === false ? false : (args.useMemory ?? true);
 
+        // Add client-side tools from the caller
+        if (clientSideTools && Array.isArray(clientSideTools) && clientSideTools.length > 0) {
+            logger.info(`Adding ${clientSideTools.length} client-side tools from caller`);
+            clientSideTools.forEach(tool => {
+                const toolName = tool.function?.name?.toLowerCase();
+                if (toolName) {
+                    // Mark as client-side tool and add to available tools
+                    entityTools[toolName] = {
+                        definition: {
+                            ...tool,
+                            clientSide: true,  // Mark it as client-side
+                            icon: tool.icon || '📱'
+                        },
+                        pathwayName: 'client_side_execution',  // Placeholder pathway
+                        clientSide: true
+                    };
+                    entityToolsOpenAiFormat.push(tool);
+                    logger.info(`Registered client-side tool: ${toolName}`);
+                }
+            });
+        }
+
         // Initialize chat history if needed
         if (!args.chatHistory || args.chatHistory.length === 0) {
             args.chatHistory = [];

package/server/clientToolCallbacks.js

@@ -0,0 +1,241 @@
+// clientToolCallbacks.js
+// Storage and management for pending client-side tool callbacks
+
+import logger from '../lib/logger.js';
+import Redis from 'ioredis';
+import { config } from '../config.js';
+
+// Map to store pending client tool callbacks
+// Key: toolCallbackId, Value: { resolve, reject, timeout, requestId }
+const pendingCallbacks = new Map();
+
+// Default timeout for client tool responses (5 minutes)
+// Increased from 60s to 5min to accommodate longer operations like CreateApplet
+const DEFAULT_TIMEOUT = 300000;
+
+// Redis setup for cross-instance communication
+const connectionString = config.get('storageConnectionString');
+const clientToolCallbackChannel = 'clientToolCallbacks';
+
+let subscriptionClient;
+let publisherClient;
+
+if (connectionString) {
+    logger.info(`Setting up Redis pub/sub for client tool callbacks on channel: ${clientToolCallbackChannel}`);
+    
+    try {
+        subscriptionClient = new Redis(connectionString);
+        subscriptionClient.on('error', (error) => {
+            logger.error(`Redis subscriptionClient error (clientToolCallbacks): ${error}`);
+        });
+        
+        subscriptionClient.on('connect', () => {
+            subscriptionClient.subscribe(clientToolCallbackChannel, (error) => {
+                if (error) {
+                    logger.error(`Error subscribing to Redis channel ${clientToolCallbackChannel}: ${error}`);
+                } else {
+                    logger.info(`Subscribed to client tool callback channel: ${clientToolCallbackChannel}`);
+                }
+            });
+        });
+        
+        subscriptionClient.on('message', (channel, message) => {
+            if (channel === clientToolCallbackChannel) {
+                try {
+                    const { toolCallbackId, result } = JSON.parse(message);
+                    logger.debug(`Received client tool callback via Redis: ${toolCallbackId}`);
+                    
+                    // Try to resolve it locally (will only work if this instance has the pending callback)
+                    resolveClientToolCallbackLocal(toolCallbackId, result);
+                } catch (error) {
+                    logger.error(`Error processing client tool callback from Redis: ${error}`);
+                }
+            }
+        });
+    } catch (error) {
+        logger.error(`Redis connection error (clientToolCallbacks): ${error}`);
+    }
+    
+    try {
+        publisherClient = new Redis(connectionString);
+        publisherClient.on('error', (error) => {
+            logger.error(`Redis publisherClient error (clientToolCallbacks): ${error}`);
+        });
+    } catch (error) {
+        logger.error(`Redis connection error (clientToolCallbacks): ${error}`);
+    }
+} else {
+    logger.info('No Redis connection configured. Client tool callbacks will only work on single instance.');
+}
+
+/**
+ * Register a pending client tool callback
+ * @param {string} toolCallbackId - Unique ID for this tool call
+ * @param {string} requestId - The request ID for logging/tracking
+ * @param {number} timeoutMs - Timeout in milliseconds
+ * @returns {Promise} Promise that resolves when client submits the result
+ */
+export function waitForClientToolResult(toolCallbackId, requestId, timeoutMs = DEFAULT_TIMEOUT) {
+    return new Promise((resolve, reject) => {
+        // Set up timeout
+        const timeout = setTimeout(() => {
+            pendingCallbacks.delete(toolCallbackId);
+            logger.error(`Client tool callback timeout for ${toolCallbackId} (requestId: ${requestId})`);
+            reject(new Error(`Client tool execution timeout after ${timeoutMs}ms`));
+        }, timeoutMs);
+
+        // Store the callback
+        pendingCallbacks.set(toolCallbackId, {
+            resolve,
+            reject,
+            timeout,
+            requestId,
+            createdAt: Date.now()
+        });
+
+        logger.info(`Registered client tool callback: ${toolCallbackId} (requestId: ${requestId})`);
+    });
+}
+
+/**
+ * Resolve a pending client tool callback locally (internal use)
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {object} result - The result from the client
+ * @returns {boolean} True if callback was found and resolved
+ */
+function resolveClientToolCallbackLocal(toolCallbackId, result) {
+    const callback = pendingCallbacks.get(toolCallbackId);
+    
+    if (!callback) {
+        // This is normal in a multi-instance setup - the callback might be on another instance
+        logger.debug(`No pending callback found for toolCallbackId: ${toolCallbackId} (may be on another instance)`);
+        return false;
+    }
+
+    // Clear the timeout
+    clearTimeout(callback.timeout);
+    
+    // Remove from pending
+    pendingCallbacks.delete(toolCallbackId);
+    
+    logger.info(`Resolved client tool callback: ${toolCallbackId} (requestId: ${callback.requestId})`);
+    
+    // Resolve the promise
+    callback.resolve(result);
+    
+    return true;
+}
+
+/**
+ * Resolve a pending client tool callback with the result
+ * This function publishes to Redis so all instances can attempt to resolve
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {object} result - The result from the client
+ * @returns {Promise<boolean>} True if callback was published/resolved
+ */
+export async function resolveClientToolCallback(toolCallbackId, result) {
+    if (publisherClient) {
+        // Publish to Redis so all instances can try to resolve
+        try {
+            const message = JSON.stringify({ toolCallbackId, result });
+            logger.debug(`Publishing client tool callback to Redis: ${toolCallbackId}`);
+            await publisherClient.publish(clientToolCallbackChannel, message);
+            return true;
+        } catch (error) {
+            logger.error(`Error publishing client tool callback to Redis: ${error}`);
+            // Fall back to local resolution
+            return resolveClientToolCallbackLocal(toolCallbackId, result);
+        }
+    } else {
+        // No Redis, resolve locally
+        return resolveClientToolCallbackLocal(toolCallbackId, result);
+    }
+}
+
+/**
+ * Reject a pending client tool callback locally (internal use)
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {Error} error - The error
+ * @returns {boolean} True if callback was found and rejected
+ */
+function rejectClientToolCallbackLocal(toolCallbackId, error) {
+    const callback = pendingCallbacks.get(toolCallbackId);
+    
+    if (!callback) {
+        logger.debug(`No pending callback found for toolCallbackId: ${toolCallbackId} (may be on another instance)`);
+        return false;
+    }
+
+    // Clear the timeout
+    clearTimeout(callback.timeout);
+    
+    // Remove from pending
+    pendingCallbacks.delete(toolCallbackId);
+    
+    logger.info(`Rejected client tool callback: ${toolCallbackId} (requestId: ${callback.requestId})`);
+    
+    // Reject the promise
+    callback.reject(error);
+    
+    return true;
+}
+
+/**
+ * Reject a pending client tool callback with an error
+ * This function publishes to Redis so all instances can attempt to reject
+ * @param {string} toolCallbackId - The tool callback ID
+ * @param {Error} error - The error
+ * @returns {Promise<boolean>} True if callback was published/rejected
+ */
+export async function rejectClientToolCallback(toolCallbackId, error) {
+    if (publisherClient) {
+        // Publish to Redis so all instances can try to reject
+        try {
+            const message = JSON.stringify({ 
+                toolCallbackId, 
+                result: { success: false, error: error.message || error.toString() } 
+            });
+            logger.debug(`Publishing client tool callback rejection to Redis: ${toolCallbackId}`);
+            await publisherClient.publish(clientToolCallbackChannel, message);
+            return true;
+        } catch (publishError) {
+            logger.error(`Error publishing client tool callback rejection to Redis: ${publishError}`);
+            // Fall back to local rejection
+            return rejectClientToolCallbackLocal(toolCallbackId, error);
+        }
+    } else {
+        // No Redis, reject locally
+        return rejectClientToolCallbackLocal(toolCallbackId, error);
+    }
+}
+
+/**
+ * Get count of pending callbacks (for monitoring)
+ */
+export function getPendingCallbackCount() {
+    return pendingCallbacks.size;
+}
+
+/**
+ * Clean up old callbacks (for maintenance)
+ */
+export function cleanupOldCallbacks(maxAgeMs = 120000) {
+    const now = Date.now();
+    let cleaned = 0;
+    
+    for (const [id, callback] of pendingCallbacks.entries()) {
+        if (now - callback.createdAt > maxAgeMs) {
+            clearTimeout(callback.timeout);
+            pendingCallbacks.delete(id);
+            callback.reject(new Error('Callback expired during cleanup'));
+            cleaned++;
+        }
+    }
+    
+    if (cleaned > 0) {
+        logger.info(`Cleaned up ${cleaned} old client tool callbacks`);
+    }
+    
+    return cleaned;
+}
+

package/server/executeWorkspace.js

@@ -80,6 +80,7 @@ const resolveAndAddFileContent = async (pathways, pathwayArgs, requestId, config
 
 // Helper function to execute pathway with cortex pathway name or fallback to legacy
 const executePathwayWithFallback = async (pathway, pathwayArgs, contextValue, info, requestId, originalPrompt = null, config) => {
+    // Extract cortexPathwayName from originalPrompt (could be original object or transformed Prompt object)
     const cortexPathwayName = (originalPrompt && typeof originalPrompt === 'object' && originalPrompt.cortexPathwayName) 
         ? originalPrompt.cortexPathwayName 
         : null;
@@ -100,6 +101,12 @@ const executePathwayWithFallback = async (pathway, pathwayArgs, contextValue, in
             // Remove old aiStyle parameter (no longer used)
             delete cortexArgs.aiStyle;
             
+            // Extract researchMode from originalPrompt if not already in pathwayArgs
+            // originalPrompt could be the original object from JSON or a transformed Prompt object
+            if (originalPrompt && typeof originalPrompt === 'object' && originalPrompt.researchMode !== undefined) {
+                cortexArgs.researchMode = originalPrompt.researchMode;
+            }
+            
             // Transform context parameters to agentContext array format (only if agentContext not already provided)
             if (!cortexArgs.agentContext && (cortexArgs.contextId || cortexArgs.contextKey || cortexArgs.altContextId || cortexArgs.altContextKey)) {
                 const agentContext = [];

package/server/graphql.js

@@ -20,7 +20,7 @@ import logger from '../lib/logger.js';
 import { buildModelEndpoints } from '../lib/requestExecutor.js';
 import { startTestServer } from '../tests/helpers/server.js';
 import { requestState } from './requestState.js';
-import { cancelRequestResolver } from './resolver.js';
+import { cancelRequestResolver, submitClientToolResultResolver } from './resolver.js';
 import subscriptions from './subscriptions.js';
 import { getMessageTypeDefs } from './typeDef.js';
 import { buildRestEndpoints } from './rest.js';
@@ -79,6 +79,7 @@ const getTypedefs = (pathways, pathwayManager) => {
 
     type Mutation {
         cancelRequest(requestId: String!): Boolean
+        submitClientToolResult(requestId: String!, toolCallbackId: String!, result: String!, success: Boolean!): Boolean
     }
 
     ${getExecuteWorkspaceTypeDefs()}
@@ -140,6 +141,7 @@ const getResolvers = (config, pathways, pathwayManager) => {
         },
         Mutation: {
             'cancelRequest': cancelRequestResolver,
+            'submitClientToolResult': submitClientToolResultResolver,
             ...mutationResolvers,
             ...pathwayManagerResolvers.Mutation
         },

package/server/plugins/gemini15VisionPlugin.js

@@ -44,8 +44,9 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                     try {
                         // First try to parse as JSON if it's a string
                         const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart;
-                        const {type, text, image_url, gcs} = part;
-                        let fileUrl = gcs || image_url?.url;
+                        const {type, text, image_url, gcs, url} = part;
+                        // Check for URL in multiple places: gcs, image_url.url, or direct url property
+                        let fileUrl = gcs || image_url?.url || url;
 
                         if (typeof part === 'string') {
                             return { text: inputPart };
@@ -72,9 +73,12 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                                 if (!base64Data) {
                                     return null;
                                 }
+                                // Extract MIME type from data URL if available
+                                const mimeMatch = fileUrl.match(/data:([^;]+);base64,/);
+                                const mimeType = mimeMatch ? mimeMatch[1] : 'image/jpeg';
                                 return {
                                     inlineData: {
-                                        mimeType: 'image/jpeg',
+                                        mimeType: mimeType,
                                         data: base64Data
                                     }
                                 };
@@ -85,6 +89,15 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                                         fileUri: fileUrl
                                     }
                                 };
+                            } else if (fileUrl.startsWith('http://') || fileUrl.startsWith('https://')) {
+                                // Gemini can read directly from HTTP/HTTPS URLs using fileData with fileUri
+                                // No need to fetch and convert to base64
+                                return {
+                                    fileData: {
+                                        mimeType: mime.lookup(fileUrl) || 'image/jpeg',
+                                        fileUri: fileUrl
+                                    }
+                                };
                             }
                             return null;
                         }

package/server/resolver.js

@@ -1,8 +1,9 @@
 import { fulfillWithTimeout } from '../lib/promiser.js';
 import { PathwayResolver } from './pathwayResolver.js';
 import CortexResponse from '../lib/cortexResponse.js';
-import { withRequestLoggingDisabled } from '../lib/logger.js';
+import logger, { withRequestLoggingDisabled } from '../lib/logger.js';
 import { sanitizeBase64 } from '../lib/util.js';
+import { resolveClientToolCallback } from './clientToolCallbacks.js';
 
 // This resolver uses standard parameters required by Apollo server:
 // (parent, args, contextValue, info)
@@ -99,6 +100,40 @@ const cancelRequestResolver = (parent, args, contextValue, _info) => {
     return true
 }
 
+const submitClientToolResultResolver = async (parent, args, contextValue, _info) => {
+    const { requestId, toolCallbackId, result, success } = args;
+    
+    logger.info(`Received client tool result submission: requestId=${requestId}, toolCallbackId=${toolCallbackId}, success=${success}`);
+    
+    try {
+        // Parse the result if it's a string
+        let parsedResult = result;
+        try {
+            parsedResult = JSON.parse(result);
+        } catch (e) {
+            // If parsing fails, use the string as-is
+        }
+        
+        // Resolve the waiting callback (now async, publishes to Redis if available)
+        const resolved = await resolveClientToolCallback(toolCallbackId, {
+            success,
+            data: parsedResult,
+            error: !success ? (parsedResult.error || 'Tool execution failed') : null
+        });
+        
+        if (!resolved) {
+            logger.warn(`Failed to publish/resolve callback for toolCallbackId: ${toolCallbackId}`);
+            return false;
+        }
+        
+        logger.info(`Successfully published/resolved client tool callback: ${toolCallbackId}`);
+        return true;
+    } catch (error) {
+        logger.error(`Error in submitClientToolResultResolver: ${error.message}`);
+        return false;
+    }
+}
+
 export {
-    resolver, rootResolver, cancelRequestResolver
+    resolver, rootResolver, cancelRequestResolver, submitClientToolResultResolver
 };

package/tests/integration/clientToolCallbacks.test.js

@@ -0,0 +1,161 @@
+// Test for client tool callbacks with Redis pub/sub
+import { describe, it, expect, beforeAll, afterAll } from '@jest/globals';
+import { 
+    waitForClientToolResult, 
+    resolveClientToolCallback,
+    getPendingCallbackCount 
+} from '../../server/clientToolCallbacks.js';
+
+describe('Client Tool Callbacks - Multi-Instance Support', () => {
+    const mockRequestId = 'test-request-123';
+    
+    afterAll(async () => {
+        // Give Redis time to clean up connections
+        await new Promise(resolve => setTimeout(resolve, 100));
+    });
+
+    it('should register and resolve a callback locally', async () => {
+        const toolCallbackId = 'test-callback-1';
+        
+        // Start waiting for result
+        const waitPromise = waitForClientToolResult(toolCallbackId, mockRequestId, 5000);
+        
+        // Verify callback is registered
+        expect(getPendingCallbackCount()).toBeGreaterThan(0);
+        
+        // Simulate client submitting result
+        const testResult = {
+            success: true,
+            data: { message: 'Test completed' },
+            error: null
+        };
+        
+        // Resolve the callback (this will publish to Redis if available, or resolve locally)
+        const resolved = await resolveClientToolCallback(toolCallbackId, testResult);
+        expect(resolved).toBe(true);
+        
+        // Wait for the result
+        const result = await waitPromise;
+        
+        expect(result).toEqual(testResult);
+        expect(result.success).toBe(true);
+        expect(result.data.message).toBe('Test completed');
+    });
+
+    it('should timeout if no result is received', async () => {
+        const toolCallbackId = 'test-callback-timeout';
+        
+        // Start waiting with a short timeout
+        const waitPromise = waitForClientToolResult(toolCallbackId, mockRequestId, 100);
+        
+        // Don't resolve - let it timeout
+        await expect(waitPromise).rejects.toThrow('Client tool execution timeout');
+    }, 10000);
+
+    it('should handle callback with error result', async () => {
+        const toolCallbackId = 'test-callback-error';
+        
+        // Start waiting for result
+        const waitPromise = waitForClientToolResult(toolCallbackId, mockRequestId, 5000);
+        
+        // Simulate client submitting error result
+        const errorResult = {
+            success: false,
+            data: null,
+            error: 'Tool execution failed'
+        };
+        
+        // Resolve with error
+        await resolveClientToolCallback(toolCallbackId, errorResult);
+        
+        // Wait for the result
+        const result = await waitPromise;
+        
+        expect(result.success).toBe(false);
+        expect(result.error).toBe('Tool execution failed');
+    });
+
+    it('should handle multiple concurrent callbacks', async () => {
+        const callbacks = [];
+        const numCallbacks = 5;
+        
+        // Register multiple callbacks
+        for (let i = 0; i < numCallbacks; i++) {
+            const callbackId = `concurrent-callback-${i}`;
+            const promise = waitForClientToolResult(callbackId, mockRequestId, 5000);
+            callbacks.push({ id: callbackId, promise });
+        }
+        
+        // Resolve all callbacks
+        for (let i = 0; i < numCallbacks; i++) {
+            const result = {
+                success: true,
+                data: { index: i, message: `Result ${i}` },
+                error: null
+            };
+            await resolveClientToolCallback(callbacks[i].id, result);
+        }
+        
+        // Wait for all results
+        const results = await Promise.all(callbacks.map(cb => cb.promise));
+        
+        // Verify all results
+        expect(results).toHaveLength(numCallbacks);
+        results.forEach((result, index) => {
+            expect(result.success).toBe(true);
+            expect(result.data.index).toBe(index);
+        });
+    });
+
+    it('should return false when resolving non-existent callback', async () => {
+        const nonExistentId = 'non-existent-callback-id';
+        
+        const result = {
+            success: true,
+            data: { message: 'Test' },
+            error: null
+        };
+        
+        // This should publish to Redis (if available) or return false locally
+        // Either way, it should not throw an error
+        const resolved = await resolveClientToolCallback(nonExistentId, result);
+        
+        // In Redis mode, this returns true (published)
+        // In local mode, this returns false (not found)
+        expect(typeof resolved).toBe('boolean');
+    });
+});
+
+describe('Client Tool Callbacks - Performance', () => {
+    it('should handle rapid callback resolution', async () => {
+        const start = Date.now();
+        const numCallbacks = 100;
+        const callbacks = [];
+        
+        // Register many callbacks rapidly
+        for (let i = 0; i < numCallbacks; i++) {
+            const callbackId = `perf-callback-${i}`;
+            const promise = waitForClientToolResult(callbackId, 'perf-test', 5000);
+            callbacks.push({ id: callbackId, promise });
+        }
+        
+        // Resolve all callbacks rapidly
+        for (let i = 0; i < numCallbacks; i++) {
+            await resolveClientToolCallback(callbacks[i].id, {
+                success: true,
+                data: { index: i },
+                error: null
+            });
+        }
+        
+        // Wait for all
+        const results = await Promise.all(callbacks.map(cb => cb.promise));
+        const duration = Date.now() - start;
+        
+        expect(results).toHaveLength(numCallbacks);
+        expect(duration).toBeLessThan(10000); // Should complete in under 10 seconds
+        
+        console.log(`Performance test: ${numCallbacks} callbacks resolved in ${duration}ms (${(duration/numCallbacks).toFixed(2)}ms avg per callback)`);
+    }, 15000);
+});
+

package/tests/integration/features/tools/fileCollection.test.js

@@ -1086,7 +1086,7 @@ test('File collection: syncAndStripFilesFromChatHistory only strips collection f
         ];
         
         // Process chat history
-        const { chatHistory: processed } = await syncAndStripFilesFromChatHistory(chatHistory, createAgentContext(contextId));
+        const { chatHistory: processed, availableFiles } = await syncAndStripFilesFromChatHistory(chatHistory, createAgentContext(contextId));
         
         // Verify only collection file was stripped
         const content = processed[0].content;

package/tests/unit/plugins/multimodal_conversion.test.js

@@ -432,9 +432,9 @@ test('Gemini 1.5 image URL type handling', t => {
             { type: 'image_url', image_url: { url: 'gs://my-bucket/image1.jpg' } },
             // Base64 URL - should be converted to inlineData
             { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,/9j/4AAQSkZJRg...' } },
-            // Regular HTTP URL - should be dropped (return null)
+            // Regular HTTP URL - should be converted to fileData (Gemini supports HTTP URLs directly)
             { type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } },
-            // Azure blob URL - should be dropped (return null)
+            // Azure blob URL - should be converted to fileData (Gemini supports HTTP URLs directly)
             { type: 'image_url', image_url: { url: 'https://myaccount.blob.core.windows.net/container/image.jpg' } }
         ]}
     ];
@@ -442,20 +442,30 @@ test('Gemini 1.5 image URL type handling', t => {
     const { modifiedMessages } = gemini15.convertMessagesToGemini(messages);
 
     t.is(modifiedMessages.length, 1);
-    t.is(modifiedMessages[0].parts.length, 3); // text + gcs + base64 (2 urls dropped)
-    
+    t.is(modifiedMessages[0].parts.length, 5); // text + gcs + base64 + http + azure (all urls kept)
+
     // Check text part
     t.is(modifiedMessages[0].parts[0].text, 'Process these images:');
-    
+
     // Check GCS URL handling
     t.true('fileData' in modifiedMessages[0].parts[1]);
     t.is(modifiedMessages[0].parts[1].fileData.fileUri, 'gs://my-bucket/image1.jpg');
     t.is(modifiedMessages[0].parts[1].fileData.mimeType, 'image/jpeg');
-    
+
     // Check base64 URL handling
     t.true('inlineData' in modifiedMessages[0].parts[2]);
     t.is(modifiedMessages[0].parts[2].inlineData.mimeType, 'image/jpeg');
     t.is(modifiedMessages[0].parts[2].inlineData.data, '/9j/4AAQSkZJRg...');
+
+    // Check HTTP URL handling
+    t.true('fileData' in modifiedMessages[0].parts[3]);
+    t.is(modifiedMessages[0].parts[3].fileData.fileUri, 'https://example.com/image.jpg');
+    t.is(modifiedMessages[0].parts[3].fileData.mimeType, 'image/jpeg');
+
+    // Check Azure blob URL handling
+    t.true('fileData' in modifiedMessages[0].parts[4]);
+    t.is(modifiedMessages[0].parts[4].fileData.fileUri, 'https://myaccount.blob.core.windows.net/container/image.jpg');
+    t.is(modifiedMessages[0].parts[4].fileData.mimeType, 'image/jpeg');
 });
 
 // Test edge cases for image URLs in Gemini 1.5

package/RELEASE_NOTES_20251231_103631.md

@@ -1,15 +0,0 @@
-### New Features
-- **Batch File Analysis**: The `AnalyzeFile` tool now supports an array of `files` for simultaneous analysis, in addition to the legacy single `file` parameter.
-- **Unified File Collection Loading**: Refactored internal file handling to use a single, unified function (`loadFileCollection`) that handles deduplication across multiple contexts (e.g., user and workspace) and consistent filtering logic.
-
-### Improvements
-- **Enhanced File Filtering**:
-    - Improved `inCollection` status logic to treat empty arrays consistently with `undefined` states.
-    - Added support for loading all files from a context regardless of their specific chat collection status, facilitating better handling of system-level uploads.
-- **Search Tool Refinements**: Updated `SearchFileCollection` descriptions to clarify that it uses simple substring matching and does not support complex boolean operators (AND/OR).
-- **Tool Documentation**: Improved descriptions for `AddFileToCollection`, `ListFileCollection`, and `SearchFileCollection` to help agents better understand chat-specific vs. global file visibility.
-- **Reference Counting**: Improved metadata synchronization to better track file usage across different chat sessions, ensuring files uploaded via external tools (like Labeeb) are correctly integrated into the collection.
-
-### Bug Fixes
-- **Media API Timeout**: Significantly increased the timeout for the media chunks API from 30 seconds to 10 minutes (600,000ms) to accommodate larger file processing and slow network conditions.
-- **Metadata Normalization**: Fixed inconsistencies in how \

package/RELEASE_NOTES_20251231_110946.md

@@ -1,5 +0,0 @@
-### Core Improvements
-- **Unified File Collection Loading**: Refactored file loading logic into a single, robust `loadFileCollection` function. This supports compound contexts (user + workspace), deduplication of files across multiple sources, and more flexible filtering.
-- **Enhanced Media Support**: Significantly increased the timeout for the media chunks API from 30 seconds to 600 seconds to better handle large file processing.
-- **Support for Untagged Uploads**: Improved synchronization logic to identify and process files that exist in storage but lack explicit collection metadata (e.g., Labeeb uploads). These files are now correctly stripped from chat history and added to the relevant chat collection upon use.
-- **Improved Filtering Logic**: Refined how the system distinguishes between global files, chat-specific files, and files not yet in any collection. Empty collection arrays are now treated consistently as \