@aj-archipelago/cortex 1.3.53 → 1.3.55

package/config.js

@@ -326,6 +326,33 @@ var config = convict({
                     "Content-Type": "application/json"
                 },
             },
+            "replicate-flux-kontext-pro": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-kontext-pro/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "replicate-flux-kontext-max": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-kontext-max/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "replicate-multi-image-kontext-max": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/flux-kontext-apps/multi-image-kontext-max/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
             "azure-video-translate": {
                 "type": "AZURE-VIDEO-TRANSLATE",
                 "url": "https://eastus.api.cognitive.microsoft.com/videotranslation",

package/lib/entityConstants.js

@@ -120,6 +120,8 @@ term~N                 (Match terms similar to "term", edit distance N)
 
     AI_GROUNDING_INSTRUCTIONS: "# Grounding Responses\n\nIf you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual :cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
 
+    AI_AVAILABLE_FILES: "# Available Files\n\nThe following files are available for you to use in your tool calls or responses:\n{{{availableFiles}}}\n",
+
     AI_MEMORY_INSTRUCTIONS: `# Memory Instructions
 
 You have a memory system that contains important details, instructions, and context. Consult your memories when formulating a response to ensure your answers reflect previous learnings and context.

package/lib/util.js

@@ -310,6 +310,60 @@ function removeImageAndFileFromMessage(message) {
     return modifiedMessage;
 }
 
+// Helper function to extract file URLs from a content object
+function extractFileUrlsFromContent(contentObj) {
+    const urls = [];
+    if (contentObj.type === 'image_url' && contentObj.image_url?.url) {
+        urls.push(contentObj.image_url.url);
+    } else if (contentObj.type === 'file' && contentObj.file) {
+        urls.push(contentObj.file);
+    }
+    return urls;
+}
+
+function getAvailableFiles(chatHistory) {
+    const availableFiles = [];
+    
+    if (!chatHistory || !Array.isArray(chatHistory)) {
+        return availableFiles;
+    }
+    
+    for (const message of chatHistory) {
+        if (!message || !message.content) {
+            continue;
+        }
+        
+        // Handle array content
+        if (Array.isArray(message.content)) {
+            for (const content of message.content) {
+                try {
+                    const contentObj = typeof content === 'string' ? JSON.parse(content) : content;
+                    availableFiles.push(...extractFileUrlsFromContent(contentObj));
+                } catch (e) {
+                    // Not JSON or couldn't be parsed, continue
+                    continue;
+                }
+            }
+        }
+        // Handle string content
+        else if (typeof message.content === 'string') {
+            try {
+                const contentObj = JSON.parse(message.content);
+                availableFiles.push(...extractFileUrlsFromContent(contentObj));
+            } catch (e) {
+                // Not JSON or couldn't be parsed, continue
+                continue;
+            }
+        }
+        // Handle object content
+        else if (typeof message.content === 'object') {
+            availableFiles.push(...extractFileUrlsFromContent(message.content));
+        }
+    }
+    
+    return availableFiles;
+}
+
 export { 
     getUniqueId,
     getSearchResultId,
@@ -322,5 +376,6 @@ export {
     alignSubtitles,
     getMediaChunks,
     markCompletedForCleanUp,
-    removeOldImageAndFileContent
+    removeOldImageAndFileContent,
+    getAvailableFiles
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.53",
+  "version": "1.3.55",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {
@@ -33,7 +33,7 @@
   "type": "module",
   "homepage": "https://github.com/aj-archipelago/cortex#readme",
   "dependencies": {
-    "@aj-archipelago/subvibe": "^1.0.10",
+    "@aj-archipelago/subvibe": "^1.0.12",
     "@apollo/server": "^4.7.3",
     "@apollo/server-plugin-response-cache": "^4.1.2",
     "@apollo/utils.keyvadapter": "^3.0.0",

package/pathways/image_flux.js

@@ -13,5 +13,7 @@ export default {
     output_format: "webp",
     output_quality: 80,
     steps: 4,
+    input_image: "", // URL to input image for models that support it
+    input_image_2: "", // URL to second input image for models that support it
   },
 };

package/pathways/system/entity/sys_entity_agent.js

@@ -5,7 +5,7 @@ const MAX_TOOL_CALLS = 50;
 import { callPathway, callTool, say } from '../../../lib/pathwayTools.js';
 import logger from '../../../lib/logger.js';
 import { config } from '../../../config.js';
-import { chatArgsHasImageUrl, removeOldImageAndFileContent } from '../../../lib/util.js';
+import { chatArgsHasImageUrl, removeOldImageAndFileContent, getAvailableFiles } from '../../../lib/util.js';
 import { Prompt } from '../../../server/prompt.js';
 import { getToolsForEntity, loadEntityConfig } from './tools/shared/sys_entity_tools.js';
 
@@ -275,7 +275,7 @@ export default {
         const instructionTemplates = entityInstructions ? (entityInstructions + '\n\n') : `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\n`;
 
         const promptMessages = [
-            {"role": "system", "content": `${promptPrefix}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_SEARCH_RULES}}\n\n{{renderTemplate AI_SEARCH_SYNTAX}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n${memoryTemplates}{{renderTemplate AI_DATETIME}}`},
+            {"role": "system", "content": `${promptPrefix}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_SEARCH_RULES}}\n\n{{renderTemplate AI_SEARCH_SYNTAX}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n${memoryTemplates}{{renderTemplate AI_AVAILABLE_FILES}}\n\n{{renderTemplate AI_DATETIME}}`},
             "{{chatHistory}}",
         ];
 
@@ -294,6 +294,8 @@ export default {
             args.chatHistory = args.chatHistory.slice(-20);
         }
 
+        const availableFiles = getAvailableFiles(args.chatHistory);
+
         // remove old image and file content
         const visionContentPresent = chatArgsHasImageUrl(args);
         visionContentPresent && (args.chatHistory = removeOldImageAndFileContent(args.chatHistory));
@@ -327,6 +329,7 @@ export default {
             let response = await runAllPrompts({
                 ...args,
                 chatHistory: currentMessages,
+                availableFiles,
                 tools: entityToolsOpenAiFormat,
                 tool_choice: memoryLookupRequired ? "required" : "auto"
             });

package/pathways/system/entity/tools/sys_tool_image.js

@@ -1,5 +1,5 @@
 // sys_tool_image.js
-// Entity tool that creates images for the entity to show to the user
+// Entity tool that creates and modifies images for the entity to show to the user
 import { callPathway } from '../../../../lib/pathwayTools.js';
 
 export default {
@@ -10,7 +10,7 @@ export default {
         model: 'oai-gpt4o',
     },
     timeout: 300,
-    toolDefinition: {
+    toolDefinition: [{
         type: "function",
         icon: "🎨",
         function: {
@@ -36,6 +36,36 @@ export default {
             }
         }
     },
+    {
+        type: "function",
+        icon: "🔄",
+        function: {
+            name: "ModifyImage",
+            description: "Use when asked to modify, transform, or edit an existing image. This tool can apply various transformations like style changes, artistic effects, or specific modifications to an image that has been previously uploaded or generated. It takes up to two input images as a reference and outputs a new image based on the instructions.",
+            parameters: {
+                type: "object",
+                properties: {
+                    inputImage: {
+                        type: "string",
+                        description: "The first image URL copied exactly from an image_url field in your chat context."
+                    },
+                    inputImage2: {
+                        type: "string",
+                        description: "The second input image URL copied exactly from an image_url field in your chat context if there is one."
+                    },
+                    detailedInstructions: {
+                        type: "string",
+                        description: "A very detailed prompt describing how you want to modify the image. Be specific about the changes you want to make, including style changes, artistic effects, or specific modifications. The more detailed and descriptive the prompt, the better the result."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["inputImage", "detailedInstructions", "userMessage"]
+            }
+        }
+    }],
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
         const pathwayResolver = resolver;
@@ -46,8 +76,27 @@ export default {
             let numberResults = args.numberResults || 1;
             let negativePrompt = args.negativePrompt || "";
 
+            // If we have an input image, use the flux-kontext-max model
+            if (args.inputImage) {
+                model = "replicate-flux-kontext-max";
+            }
+
+            // If we have two input images, use the multi-image-kontext-max model
+            if (args.inputImage2) {
+                model = "replicate-multi-image-kontext-max";
+            }
+
             pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
-            return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model, stream: false });
+            return await callPathway('image_flux', {
+                ...args, 
+                text: prompt, 
+                negativePrompt, 
+                numberResults, 
+                model, 
+                stream: false,
+                input_image: args.inputImage,
+                input_image_2: args.inputImage2,
+            });
 
         } catch (e) {
             pathwayResolver.logError(e.message ?? e);

package/server/plugins/replicateApiPlugin.js

@@ -91,6 +91,50 @@ class ReplicateApiPlugin extends ModelPlugin {
         };
         break;
       }
+      case "replicate-flux-kontext-pro":
+      case "replicate-flux-kontext-max": {
+        const validRatios = [
+          '1:1', '16:9', '21:9', '3:2', '2:3', '4:5',
+          '5:4', '3:4', '4:3', '9:16', '9:21', 'match_input_image'
+        ];
+
+        let safetyTolerance = combinedParameters.safety_tolerance || 3;
+        if(combinedParameters.input_image){
+          safetyTolerance = Math.min(safetyTolerance, 2);
+        }
+
+        requestParameters = {
+          input: {
+            prompt: modelPromptText,
+            input_image: combinedParameters.input_image,
+            aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
+            safety_tolerance: safetyTolerance,
+          },
+        };
+        break;
+      }
+      case "replicate-multi-image-kontext-max": {
+        const validRatios = [
+          '1:1', '16:9', '21:9', '3:2', '2:3', '4:5',
+          '5:4', '3:4', '4:3', '9:16', '9:21', 'match_input_image'
+        ];
+
+        let safetyTolerance = combinedParameters.safety_tolerance || 3;
+        if(combinedParameters.input_image_1 || combinedParameters.input_image) {
+          safetyTolerance = Math.min(safetyTolerance, 2);
+        }
+
+        requestParameters = {
+          input: {
+            prompt: modelPromptText,
+            input_image_1: combinedParameters.input_image_1 || combinedParameters.input_image,
+            input_image_2: combinedParameters.input_image_2,
+            aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
+            safety_tolerance: safetyTolerance,
+          },
+        };
+        break;
+      }
     }
 
     return requestParameters;