@aj-archipelago/cortex 1.3.67 → 1.4.0

package/pathways/system/entity/tools/sys_tool_google_search.js

@@ -107,7 +107,8 @@ export default {
         try {
             // Pass-through: call Google CSE with provided args
             const response = await callPathway('google_cse', { 
-                ...args
+                ...args,
+                text: args.q
             }, resolver);
 
             if (resolver.errors && resolver.errors.length > 0) {
@@ -117,7 +118,19 @@ export default {
                 return JSON.stringify({ _type: "SearchError", value: errorMessages, recoveryMessage: "This tool failed. You should try the backup tool for this function." });
             }
 
-            const parsedResponse = JSON.parse(response);
+            // Check if response is null or empty
+            if (!response) {
+                logger.error('Google CSE search returned null response');
+                return JSON.stringify({ _type: "SearchError", value: ["No response received from Google CSE"], recoveryMessage: "This tool failed. You should try the backup tool for this function." });
+            }
+
+            let parsedResponse;
+            try {
+                parsedResponse = JSON.parse(response);
+            } catch (parseError) {
+                logger.error(`Failed to parse Google CSE response: ${parseError.message}`);
+                return JSON.stringify({ _type: "SearchError", value: ["Invalid response format from Google CSE"], recoveryMessage: "This tool failed. You should try the backup tool for this function." });
+            }
 
             const results = [];
             const items = parsedResponse.items || [];

package/pathways/system/entity/tools/sys_tool_grok_x_search.js

@@ -10,8 +10,8 @@ export default {
     inputParameters: {
         text: '',
         userMessage: '',
-        includedHandles: [''],
-        excludedHandles: [''],
+        includedHandles: { type: 'array', items: { type: 'string' }, default: [] },
+        excludedHandles: { type: 'array', items: { type: 'string' }, default: [] },
         minFavorites: 0,
         minViews: 0,
         maxResults: 10
@@ -244,4 +244,4 @@ export default {
             throw e;
         }
     }
-};
+};

package/pathways/system/entity/tools/sys_tool_image.js

@@ -15,7 +15,7 @@ export default {
         icon: "🎨",
         function: {
             name: "GenerateImage",
-            description: "Use when asked to create, generate, or generate revisions of visual content. Any time the user asks you for a picture, a selfie, artwork, a drawing or if you want to illustrate something for the user, you can use this tool to generate any sort of image from cartoon to photo realistic. After you have generated the image, you must include the image in your response to show it to the user.",
+            description: "Use when asked to create, generate, or generate revisions of visual content. Any time the user asks you for a picture, a selfie, artwork, a drawing or if you want to illustrate something for the user, you can use this tool to generate any sort of image from cartoon to photo realistic. This tool does not display the image to the user - you need to do that with markdown in your response.",
             parameters: {
                 type: "object",
                 properties: {
@@ -23,16 +23,12 @@ export default {
                         type: "string",
                         description: "A very detailed prompt describing the image you want to create. You should be very specific - explaining subject matter, style, and details about the image including things like camera angle, lens types, lighting, photographic techniques, etc. Any details you can provide to the image creation engine will help it create the most accurate and useful images. The more detailed and descriptive the prompt, the better the result."
                     },
-                    renderText: {
-                        type: "boolean",
-                        description: "Set to true if the image should be optimized to show correct text. This is useful when the user asks for a picture of something that includes specific text as it invokes a different image generation model that is optimized for including text."
-                    },
                     userMessage: {
                         type: "string",
                         description: "A user-friendly message that describes what you're doing with this tool"
                     }
                 },
-                required: ["detailedInstructions", "renderText", "userMessage"]
+                required: ["detailedInstructions", "userMessage"]
             }
         }
     },
@@ -41,7 +37,7 @@ export default {
         icon: "🔄",
         function: {
             name: "ModifyImage",
-            description: "Use when asked to modify, transform, or edit an existing image. This tool can apply various transformations like style changes, artistic effects, or specific modifications to an image that has been previously uploaded or generated. It takes up to two input images as a reference and outputs a new image based on the instructions.",
+            description: "Use when asked to modify, transform, or edit an existing image. This tool can apply various transformations like style changes, artistic effects, or specific modifications to an image that has been previously uploaded or generated. It takes up to two input images as a reference and outputs a new image based on the instructions. This tool does not display the image to the user - you need to do that with markdown in your response.",
             parameters: {
                 type: "object",
                 properties: {
@@ -53,6 +49,10 @@ export default {
                         type: "string",
                         description: "The second input image URL copied exactly from an image_url field in your chat context if there is one."
                     },
+                    inputImage3: {
+                        type: "string",
+                        description: "The third input image URL copied exactly from an image_url field in your chat context if there is one."
+                    },
                     detailedInstructions: {
                         type: "string",
                         description: "A very detailed prompt describing how you want to modify the image. Be specific about the changes you want to make, including style changes, artistic effects, or specific modifications. The more detailed and descriptive the prompt, the better the result."
@@ -71,32 +71,37 @@ export default {
         const pathwayResolver = resolver;
 
         try {   
-            let model = "replicate-flux-11-pro";
+            let model = "replicate-seedream-4";
             let prompt = args.detailedInstructions || "";
-            let numberResults = args.numberResults || 1;
-            let negativePrompt = args.negativePrompt || "";
 
             // If we have an input image, use the flux-kontext-max model
-            if (args.inputImage) {
-                model = "replicate-flux-kontext-max";
-            }
-
-            // If we have two input images, use the multi-image-kontext-max model
-            if (args.inputImage2) {
-                model = "replicate-multi-image-kontext-max";
+            if (args.inputImage || args.inputImage2 || args.inputImage3) {
+                model = "replicate-qwen-image-edit-plus";
             }
 
             pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
-            return await callPathway('image_flux', {
+            
+            // Build parameters object, only including image parameters if they have non-empty values
+            const params = {
                 ...args, 
                 text: prompt, 
-                negativePrompt, 
-                numberResults, 
                 model, 
                 stream: false,
-                input_image: args.inputImage,
-                input_image_2: args.inputImage2,
-            });
+            };
+            
+            if (args.inputImage && args.inputImage.trim()) {
+                params.input_image = args.inputImage;
+            }
+            if (args.inputImage2 && args.inputImage2.trim()) {
+                params.input_image_2 = args.inputImage2;
+            }
+            if (args.inputImage3 && args.inputImage3.trim()) {
+                params.input_image_3 = args.inputImage3;
+            }
+            
+            // Call appropriate pathway based on model
+            const pathwayName = model.includes('seedream') ? 'image_seedream4' : 'image_qwen';
+            return await callPathway(pathwayName, params);
 
         } catch (e) {
             pathwayResolver.logError(e.message ?? e);

package/pathways/system/entity/tools/sys_tool_image_gemini.js

@@ -0,0 +1,135 @@
+// sys_tool_image_gemini.js
+// Entity tool that creates and modifies images for the entity to show to the user
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { uploadImageToCloud } from '../../../../lib/util.js';
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    /*
+    toolDefinition: [{
+        type: "function",
+        icon: "🎨",
+        function: {
+            name: "GenerateImage",
+            description: "Use when asked to create, generate, or generate revisions of visual content. Any time the user asks you for a picture, a selfie, artwork, a drawing or if you want to illustrate something for the user, you can use this tool to generate any sort of image from cartoon to photo realistic. After you have generated the image, you must include the image in your response to show it to the user.",
+            parameters: {
+                type: "object",
+                properties: {
+                    detailedInstructions: {
+                        type: "string",
+                        description: "A very detailed prompt describing the image you want to create. You should be very specific - explaining subject matter, style, and details about the image including things like camera angle, lens types, lighting, photographic techniques, etc. Any details you can provide to the image creation engine will help it create the most accurate and useful images. The more detailed and descriptive the prompt, the better the result."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["detailedInstructions", "renderText", "userMessage"]
+            }
+        }
+    },
+    {
+        type: "function",
+        icon: "🔄",
+        function: {
+            name: "ModifyImage",
+            description: "Use when asked to modify, transform, or edit an existing image. This tool can apply various transformations like style changes, artistic effects, or specific modifications to an image that has been previously uploaded or generated. It takes up to two input images as a reference and outputs a new image based on the instructions.",
+            parameters: {
+                type: "object",
+                properties: {
+                    inputImage: {
+                        type: "string",
+                        description: "The first image URL copied exactly from an image_url field in your chat context."
+                    },
+                    inputImage2: {
+                        type: "string",
+                        description: "The second input image URL copied exactly from an image_url field in your chat context if there is one."
+                    },
+                    inputImage3: {
+                        type: "string",
+                        description: "The third input image URL copied exactly from an image_url field in your chat context if there is one."
+                    },
+                    detailedInstructions: {
+                        type: "string",
+                        description: "A very detailed prompt describing how you want to modify the image. Be specific about the changes you want to make, including style changes, artistic effects, or specific modifications. The more detailed and descriptive the prompt, the better the result."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["inputImage", "detailedInstructions", "userMessage"]
+            }
+        }
+    }],
+    */
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const pathwayResolver = resolver;
+
+        try {   
+            let model = "gemini-25-flash-image";
+            let prompt = args.detailedInstructions || "";
+            
+            // Call the image generation pathway
+            let result = await callPathway('image_gemini_25', {
+                ...args, 
+                text: prompt,
+                model, 
+                stream: false,
+                input_image: args.inputImage,
+                input_image_2: args.inputImage2,
+                input_image_3: args.inputImage3,
+                optimizePrompt: true,
+            }, pathwayResolver);
+
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
+
+            if (pathwayResolver.pathwayResultData) {
+                if (pathwayResolver.pathwayResultData.artifacts && Array.isArray(pathwayResolver.pathwayResultData.artifacts)) {
+                    const uploadedImages = [];
+                    
+                    // Process each image artifact
+                    for (const artifact of pathwayResolver.pathwayResultData.artifacts) {
+                        if (artifact.type === 'image' && artifact.data && artifact.mimeType) {
+                            try {
+                                // Upload image to cloud storage
+                                const imageUrl = await uploadImageToCloud(artifact.data, artifact.mimeType, pathwayResolver);
+                                uploadedImages.push({
+                                    type: 'image',
+                                    url: imageUrl,
+                                    mimeType: artifact.mimeType
+                                });
+                            } catch (uploadError) {
+                                pathwayResolver.logError(`Failed to upload artifact: ${uploadError.message}`);
+                                // Keep original artifact as fallback
+                                uploadedImages.push(artifact);
+                            }
+                        } else {
+                            // Keep non-image artifacts as-is
+                            uploadedImages.push(artifact);
+                        }
+                    }
+                    
+                    // Return the urls of the uploaded images as text in the result
+                    result = result + '\n' + uploadedImages.map(image => image.url).join('\n');
+                }
+            } else {
+                // If result is not a CortexResponse, log a warning but return as-is
+                pathwayResolver.logWarning('No artifacts to upload');
+                result = result + '\n' + 'No images generated';
+            }
+
+            return result;
+
+        } catch (e) {
+            pathwayResolver.logError(e.message ?? e);
+            return await callPathway('sys_generator_error', { ...args, text: e.message }, pathwayResolver);
+        }
+    }
+};

package/server/graphql.js

@@ -36,10 +36,17 @@ const getPlugins = (config) => {
     //if cache is enabled and Redis is available, use it
     let cache;
     if (config.get('enableGraphqlCache') && config.get('storageConnectionString')) {
-        cache = new KeyvAdapter(new Keyv(config.get('storageConnectionString'), {
+        const keyvCache = new Keyv(config.get('storageConnectionString'), {
             ssl: true,
             abortConnect: false,
-        }));
+        });
+        
+        // Handle Redis connection errors to prevent crashes
+        keyvCache.on('error', (error) => {
+            logger.error(`GraphQL Keyv Redis connection error: ${error}`);
+        });
+        
+        cache = new KeyvAdapter(keyvCache);
         //caching similar strings, embedding hashing, ... #delta similarity 
         // TODO: custom cache key:
         // https://www.apollographql.com/docs/apollo-server/performance/cache-backends#implementing-your-own-cache-backend

package/server/modelExecutor.js

@@ -21,6 +21,7 @@ import GeminiChatPlugin from './plugins/geminiChatPlugin.js';
 import GeminiVisionPlugin from './plugins/geminiVisionPlugin.js';
 import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
 import Gemini15VisionPlugin from './plugins/gemini15VisionPlugin.js';
+import Gemini25ImagePlugin from './plugins/gemini25ImagePlugin.js';
 import AzureBingPlugin from './plugins/azureBingPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
 import NeuralSpacePlugin from './plugins/neuralSpacePlugin.js';
@@ -103,6 +104,9 @@ class ModelExecutor {
             case 'GEMINI-1.5-VISION':
                 plugin = new Gemini15VisionPlugin(pathway, model);
                 break;
+            case 'GEMINI-2.5-IMAGE':
+                plugin = new Gemini25ImagePlugin(pathway, model);
+                break;
             case 'AZURE-BING':
                 plugin = new AzureBingPlugin(pathway, model);
                 break;

package/server/pathwayResolver.js

@@ -8,8 +8,8 @@ import { Prompt } from './prompt.js';
 import { getv, setv } from '../lib/keyValueStorageClient.js';
 import { requestState } from './requestState.js';
 import { callPathway, addCitationsToResolver } from '../lib/pathwayTools.js';
-import { publishRequestProgress } from '../lib/redisSubscription.js';
 import logger from '../lib/logger.js';
+import { publishRequestProgress } from '../lib/redisSubscription.js';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { createParser } from 'eventsource-parser';
 import CortexResponse from '../lib/cortexResponse.js';
@@ -158,6 +158,11 @@ class PathwayResolver {
             return;
         }
 
+        // Handle CortexResponse objects - merge them into pathwayResultData
+        if (responseData && typeof responseData === 'object' && responseData.constructor && responseData.constructor.name === 'CortexResponse') {
+            this.pathwayResultData = this.mergeResultData(responseData);
+        }
+
         // If the response is a stream, handle it as streaming response
         if (responseData && typeof responseData.on === 'function') {
             await this.handleStream(responseData);
@@ -205,7 +210,8 @@ class PathwayResolver {
                 toolCalls: cortexResponse.toolCalls,
                 functionCall: cortexResponse.functionCall,
                 usage: cortexResponse.usage,
-                finishReason: cortexResponse.finishReason
+                finishReason: cortexResponse.finishReason,
+                artifacts: cortexResponse.artifacts
             };
             newData = cortexData;
         }
@@ -214,7 +220,7 @@ class PathwayResolver {
         const merged = { ...currentData, ...newData };
 
         // Handle array fields that should be concatenated
-        const arrayFields = ['citations', 'toolCalls'];
+        const arrayFields = ['citations', 'toolCalls', 'artifacts'];
         for (const field of arrayFields) {
             const currentArray = currentData[field] || [];
             const newArray = newData[field] || [];
@@ -593,7 +599,7 @@ class PathwayResolver {
                     if (previousResult) {
                         previousResult = this.truncate(previousResult, 2 * this.chunkMaxTokenLength);
                     }
-                    result = await this.applyPrompt(this.prompts[i], null, currentParameters);
+                    result = await this.applyPrompt(this.prompts[i], text, currentParameters);
                 } else {
                     // Limit context to N characters
                     if (previousResult) {
@@ -664,20 +670,15 @@ class PathwayResolver {
         }
         let result = '';
 
-        // If this text is empty, skip applying the prompt as it will likely be a nonsensical result
-        if (!/^\s*$/.test(text) || parameters?.file || parameters?.inputVector || this?.modelName.includes('cognitive')) {
-            result = await this.modelExecutor.execute(text, { 
-                ...parameters, 
-                ...this.savedContext,
-                memorySelf: this.memorySelf,
-                memoryDirectives: this.memoryDirectives,
-                memoryTopics: this.memoryTopics,
-                memoryUser: this.memoryUser,
-                memoryContext: this.memoryContext
-            }, prompt, this);
-        } else {
-            result = text;
-        }
+        result = await this.modelExecutor.execute(text, { 
+            ...parameters, 
+            ...this.savedContext,
+            memorySelf: this.memorySelf,
+            memoryDirectives: this.memoryDirectives,
+            memoryTopics: this.memoryTopics,
+            memoryUser: this.memoryUser,
+            memoryContext: this.memoryContext
+        }, prompt, this);
         
         requestState[this.requestId].completedCount++;
 

package/server/plugins/claude3VertexPlugin.js

@@ -304,12 +304,17 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     requestParameters.messages = modifiedMessages;
 
     // Convert OpenAI tools format to Claude format if present
-    if (typeof parameters.tools === 'string') {
-      parameters.tools = JSON.parse(parameters.tools);
+    let toolsArray = parameters.tools;
+    if (typeof toolsArray === 'string') {
+      try {
+        toolsArray = JSON.parse(toolsArray);
+      } catch (e) {
+        toolsArray = [];
+      }
     }
     
-    if (parameters.tools) {
-      requestParameters.tools = parameters.tools.map(tool => {
+    if (toolsArray && Array.isArray(toolsArray) && toolsArray.length > 0) {
+      requestParameters.tools = toolsArray.map(tool => {
         if (tool.type === 'function') {
           return {
             name: tool.function.name,
@@ -325,14 +330,13 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
       });
     }
 
+    // Handle tool_choice parameter conversion from OpenAI format to Claude format
     if (parameters.tool_choice) {
-      // Convert OpenAI tool_choice format to Claude format
       let toolChoice = parameters.tool_choice;
       
-      // Handle JSON string from REST endpoint
+      // Parse JSON string if needed
       if (typeof toolChoice === 'string') {
         try {
-          // Try to parse as JSON first
           toolChoice = JSON.parse(toolChoice);
         } catch (e) {
           // If not JSON, handle as simple string values: auto, required, none
@@ -343,11 +347,12 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
           } else if (toolChoice === 'none') {
             requestParameters.tool_choice = { type: 'none' };
           }
+          toolChoice = null; // Prevent further processing
         }
       }
       
       // Handle parsed object
-      if (toolChoice.type === "function") {
+      if (toolChoice && toolChoice.type === "function") {
         // Handle function-specific tool choice
         requestParameters.tool_choice = {
           type: "tool",

package/server/plugins/gemini15ChatPlugin.js

@@ -138,16 +138,16 @@ class Gemini15ChatPlugin extends ModelPlugin {
         }
     
         const requestParameters = {
-        contents: requestMessages,
-        generationConfig: {
-            temperature: this.temperature || 0.7,
-            maxOutputTokens: max_tokens || this.getModelMaxReturnTokens(),
-            topP: parameters.topP || 0.95,
-            topK: parameters.topK || 40,
-        },
-        ...(geminiSafetySettings ? {safety_settings: geminiSafetySettings} : {}),
-        ...(system ? {systemInstruction: system} : {}),
-        ...(geminiTools ? {tools: geminiTools} : {})
+            contents: requestMessages,
+            generationConfig: {
+                temperature: this.temperature || 0.7,
+                maxOutputTokens: max_tokens || this.getModelMaxReturnTokens(),
+                topP: parameters.topP || 0.95,
+                topK: parameters.topK || 40,
+            },
+            ...(geminiSafetySettings ? {safety_settings: geminiSafetySettings} : {}),
+            ...(system ? {systemInstruction: system} : {}),
+            ...(geminiTools ? {tools: geminiTools} : {})
         };
     
         return requestParameters;
@@ -155,6 +155,11 @@ class Gemini15ChatPlugin extends ModelPlugin {
 
     // Parse the response from the new Chat API
     parseResponse(data) {
+        // Handle CortexResponse objects by returning them as-is
+        if (data && typeof data === 'object' && data.constructor && data.constructor.name === 'CortexResponse') {
+            return data;
+        }
+
         // If data is not an array, return it directly
         let dataToMerge = [];
         if (data && data.contents && Array.isArray(data.contents)) {

package/server/plugins/gemini15VisionPlugin.js

@@ -204,34 +204,13 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
             }
         }
         
-        if (toolsArray && Array.isArray(toolsArray)) {
+        if (toolsArray && Array.isArray(toolsArray) && toolsArray.length > 0) {
             convertedTools = this.convertOpenAIToolsToGemini(toolsArray);
         }
 
-        if (cortexRequest?.tools && Array.isArray(cortexRequest.tools)) {
-            const requestTools = this.convertOpenAIToolsToGemini(cortexRequest.tools);
-            convertedTools = [...convertedTools, ...requestTools];
-        }
-
-        if (cortexRequest?.pathway?.tools && Array.isArray(cortexRequest.pathway.tools)) {
-            const pathwayTools = this.convertOpenAIToolsToGemini(cortexRequest.pathway.tools);
-            convertedTools = [...convertedTools, ...pathwayTools];
-        }
-
-        // Temporarily remove geminiTools from pathway to prevent override
-        const originalGeminiTools = cortexRequest?.pathway?.geminiTools;
-        if (cortexRequest?.pathway) {
-            delete cortexRequest.pathway.geminiTools;
-        }
-
         const baseParameters = super.getRequestParameters(text, parameters, prompt, cortexRequest);
 
-        // Restore original geminiTools
-        if (cortexRequest?.pathway && originalGeminiTools !== undefined) {
-            cortexRequest.pathway.geminiTools = originalGeminiTools;
-        }
-
-        if (convertedTools.length > 0) {
+        if (convertedTools[0]?.functionDeclarations?.length > 0) {
             baseParameters.tools = convertedTools;
             
             // Handle tool_choice parameter - convert OpenAI format to Gemini toolConfig