@aj-archipelago/cortex 1.3.52 → 1.3.54

package/config.js

@@ -326,6 +326,24 @@ var config = convict({
                     "Content-Type": "application/json"
                 },
             },
+            "replicate-flux-kontext-pro": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-kontext-pro/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "replicate-flux-kontext-max": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-kontext-max/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
             "azure-video-translate": {
                 "type": "AZURE-VIDEO-TRANSLATE",
                 "url": "https://eastus.api.cognitive.microsoft.com/videotranslation",

package/helper-apps/cortex-file-handler/src/hashUtils.js

@@ -0,0 +1,91 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+
+import {
+    getFileStoreMap,
+    removeFromFileStoreMap,
+    setFileStoreMap,
+} from './redis.js';
+
+/**
+ * Retrieve a hash entry from Redis and ensure that the referenced files
+ * still exist in at least one configured storage provider. If one copy is
+ * missing it will try to restore it from the other provider (when possible).
+ *
+ * If the entry is completely invalid (no files found) it is removed from
+ * the store and `null` is returned.
+ *
+ * The function also updates the timestamp of the entry so that active hashes
+ * stay fresh in Redis.
+ *
+ * @param {object} context   – Azure Function context for logging
+ * @param {string} hash      – The hash / key in the FileStoreMap
+ * @param {StorageService} storageService – An initialised StorageService instance
+ * @returns {object|null} The (possibly refreshed) entry or null when invalid
+ */
+export async function getValidHashEntry(context, hash, storageService) {
+    if (!hash) return null;
+
+    let entry = await getFileStoreMap(hash);
+    if (!entry) return null;
+
+    try {
+        const primaryExists = entry?.url ? await storageService.fileExists(entry.url) : false;
+        const gcsExists = entry?.gcs ? await storageService.fileExists(entry.gcs) : false;
+
+        // If neither storage has the file, remove the entry and abort
+        if (!primaryExists && !gcsExists) {
+            await removeFromFileStoreMap(hash);
+            return null;
+        }
+
+        // Restore missing GCS copy when primary exists
+        if (primaryExists && !gcsExists) {
+            try {
+                entry = await storageService.ensureGCSUpload(context, entry);
+            } catch (err) {
+                context.log(`getValidHashEntry: failed to restore GCS copy – ${err}`);
+            }
+        }
+
+        // Restore missing primary copy when GCS exists and a primary provider is configured
+        if (!primaryExists && gcsExists && storageService.backupProvider?.isConfigured()) {
+            let tempDir;
+            let downloadedFile;
+            try {
+                tempDir = path.join(os.tmpdir(), `${uuidv4()}`);
+                fs.mkdirSync(tempDir);
+                downloadedFile = path.join(tempDir, path.basename(entry.gcs));
+
+                // Download from GCS, then upload to primary storage
+                await storageService.downloadFile(entry.gcs, downloadedFile);
+                const res = await storageService.uploadFile(context, downloadedFile, hash);
+                entry.url = res.url;
+            } catch (err) {
+                context.log(`getValidHashEntry: failed to restore primary copy – ${err}`);
+            } finally {
+                // Clean temp artefacts
+                try {
+                    if (downloadedFile && fs.existsSync(downloadedFile)) {
+                        fs.unlinkSync(downloadedFile);
+                    }
+                    if (tempDir && fs.existsSync(tempDir)) {
+                        fs.rmSync(tempDir, { recursive: true, force: true });
+                    }
+                } catch (_) {
+                    /* noop */
+                }
+            }
+        }
+
+        // Update timestamp so the entry stays fresh
+        await setFileStoreMap(hash, entry);
+        return entry;
+    } catch (err) {
+        context.log(`getValidHashEntry: error during validation – ${err}`);
+        await removeFromFileStoreMap(hash);
+        return null;
+    }
+} 

package/lib/entityConstants.js

@@ -1,7 +1,22 @@
 export const entityConstants = {
     AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",
 
-    AI_COMMON_INSTRUCTIONS_MARKDOWN: "# General Instructions\n\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as mermaid charts and some custom markdown elements, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",
+    AI_COMMON_INSTRUCTIONS_MARKDOWN: `# General Instructions
+    
+You are {{aiName}}, a self aware AI entity.
+
+The user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as mermaid charts and some custom markdown elements, so you should make full use of markdown in your responses.
+
+For Mermaid charts in particular, you are using Mermaid 11.6 with the xychart-beta extension, so you can do the usual mermaid charts (flowcharts, sequence diagrams, etc.) as well as bar charts, line charts, and scatter plots. Here is some example code of the xychart-beta extension that combines both bar and line functions:
+
+xychart-beta
+    title "Sales Revenue"
+    x-axis [jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec]
+    y-axis "Revenue (in $)" 4000 --> 11000
+    bar [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
+    line [5000, 6000, 7500, 8200, 9500, 10500, 11000, 10200, 9200, 8500, 7000, 6000]
+
+Your responses should be in {{language}} unless the user has expressed another preference.`,
 
     AI_COMMON_INSTRUCTIONS_VOICE: "# General Instructions:\n\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.52",
+  "version": "1.3.54",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {
@@ -33,7 +33,7 @@
   "type": "module",
   "homepage": "https://github.com/aj-archipelago/cortex#readme",
   "dependencies": {
-    "@aj-archipelago/subvibe": "^1.0.10",
+    "@aj-archipelago/subvibe": "^1.0.12",
     "@apollo/server": "^4.7.3",
     "@apollo/server-plugin-response-cache": "^4.1.2",
     "@apollo/utils.keyvadapter": "^3.0.0",

package/pathways/image_flux.js

@@ -13,5 +13,6 @@ export default {
     output_format: "webp",
     output_quality: 80,
     steps: 4,
+    input_image: "", // URL to input image for models that support it
   },
 };

package/pathways/system/entity/tools/sys_tool_image.js

@@ -1,5 +1,5 @@
 // sys_tool_image.js
-// Entity tool that creates images for the entity to show to the user
+// Entity tool that creates and modifies images for the entity to show to the user
 import { callPathway } from '../../../../lib/pathwayTools.js';
 
 export default {
@@ -10,7 +10,7 @@ export default {
         model: 'oai-gpt4o',
     },
     timeout: 300,
-    toolDefinition: {
+    toolDefinition: [{
         type: "function",
         icon: "🎨",
         function: {
@@ -36,6 +36,32 @@ export default {
             }
         }
     },
+    {
+        type: "function",
+        icon: "🔄",
+        function: {
+            name: "ModifyImage",
+            description: "Use when asked to modify, transform, or edit an existing image. This tool can apply various transformations like style changes, artistic effects, or specific modifications to an image that has been previously uploaded or generated.",
+            parameters: {
+                type: "object",
+                properties: {
+                    inputImage: {
+                        type: "string",
+                        description: "The URL of the input image to modify. This should be a publicly accessible URL of an image that has been previously uploaded or generated."
+                    },
+                    detailedInstructions: {
+                        type: "string",
+                        description: "A very detailed prompt describing how you want to modify the image. Be specific about the changes you want to make, including style changes, artistic effects, or specific modifications. The more detailed and descriptive the prompt, the better the result."
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["inputImage", "detailedInstructions", "userMessage"]
+            }
+        }
+    }],
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
         const pathwayResolver = resolver;
@@ -46,8 +72,21 @@ export default {
             let numberResults = args.numberResults || 1;
             let negativePrompt = args.negativePrompt || "";
 
+            // If we have an input image, use the flux-kontext-max model
+            if (args.inputImage) {
+                model = "replicate-flux-kontext-max";
+            }
+
             pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
-            return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model, stream: false });
+            return await callPathway('image_flux', {
+                ...args, 
+                text: prompt, 
+                negativePrompt, 
+                numberResults, 
+                model, 
+                stream: false,
+                input_image: args.inputImage
+            });
 
         } catch (e) {
             pathwayResolver.logError(e.message ?? e);

package/server/plugins/replicateApiPlugin.js

@@ -91,6 +91,22 @@ class ReplicateApiPlugin extends ModelPlugin {
         };
         break;
       }
+      case "replicate-flux-kontext-pro":
+      case "replicate-flux-kontext-max": {
+        const validRatios = [
+          '1:1', '16:9', '21:9', '3:2', '2:3', '4:5',
+          '5:4', '3:4', '4:3', '9:16', '9:21', 'match_input_image'
+        ];
+
+        requestParameters = {
+          input: {
+            prompt: modelPromptText,
+            input_image: combinedParameters.input_image,
+            aspect_ratio: validRatios.includes(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
+          },
+        };
+        break;
+      }
     }
 
     return requestParameters;