@aj-archipelago/cortex 1.3.66 → 1.4.0

package/helper-apps/cortex-file-handler/tests/getOperations.test.js

@@ -232,20 +232,32 @@ test.serial("should fetch remote file", async (t) => {
   const requestId = uuidv4();
   const remoteUrl = "https://example.com/test.txt";
 
+  // Mock external HEAD and GET to avoid network dependency
+  const body = "hello from example";
+  const scope = nock("https://example.com")
+    .head("/test.txt")
+    .reply(200, "", {
+      "Content-Type": "text/plain",
+      "Content-Length": body.length.toString(),
+    })
+    .get("/test.txt")
+    .reply(200, body, {
+      "Content-Type": "text/plain",
+      "Content-Length": body.length.toString(),
+    });
+
   const response = await axios.get(baseUrl, {
     params: {
       fetch: remoteUrl,
       requestId,
     },
     validateStatus: (status) => true,
+    timeout: 10000,
   });
 
-  t.is(response.status, 400, "Should reject invalid URL");
-  t.is(
-    response.data,
-    "Invalid or inaccessible URL",
-    "Should return correct error message",
-  );
+  t.is(response.status, 200, "Should fetch and store remote file");
+  t.truthy(response.data.url, "Should return file URL");
+  t.true(scope.isDone(), "All external requests should be mocked and used");
 });
 
 // Test: Redis caching behavior for remote files
@@ -296,7 +308,7 @@ test.serial("should cache remote files in Redis", async (t) => {
 
     t.is(secondResponse.status, 200, "Should return cached file from Redis");
     t.truthy(secondResponse.data.url, "Should return cached file URL");
-    
+
     // Cleanup
     await cleanupHashAndFile(hash, secondResponse.data.url, baseUrl);
   } finally {

package/lib/encodeCache.js

@@ -9,6 +9,11 @@ class EncodeCache {
     }
     
     encode(value) {
+        // Handle CortexResponse objects by extracting the text content
+        if (value && typeof value === 'object' && value.constructor && value.constructor.name === 'CortexResponse') {
+            value = value.output_text || '';
+        }
+        
         if (this.encodeCache.get(value) !== -1) {
             return this.encodeCache.get(value);
         }

package/lib/keyValueStorageClient.js

@@ -30,6 +30,11 @@ const keyValueStorageClient = new Keyv(storageConnectionString, {
     namespace: `${cortexId}-cortex-context`
 });
 
+// Handle Redis connection errors to prevent crashes
+keyValueStorageClient.on('error', (error) => {
+    logger.error(`Keyv Redis connection error: ${error}`);
+});
+
 // Set values to keyv
 async function setv(key, value) {
     return keyValueStorageClient && (await keyValueStorageClient.set(key, value));

package/lib/logger.js

@@ -37,9 +37,9 @@ const getTransport = () => {
       case 'production':
         return new winston.transports.Console({ level: 'info', format: winston.format.combine(suppressNonErrorFormat(), prodFormat) });
       case 'development':
-      case 'test':
         return new winston.transports.Console({ level: 'verbose', format: winston.format.combine(suppressNonErrorFormat(), debugFormat) });
       case 'debug':
+      case 'test':
         return new winston.transports.Console({ level: 'debug', format: winston.format.combine(suppressNonErrorFormat(), debugFormat) });
       default:
         // Default to development settings if NODE_ENV is not set or unknown

package/lib/pathwayManager.js

@@ -338,36 +338,58 @@ class PathwayManager {
 
   /**
    * Creates a Prompt object from a prompt item and system prompt.
-   * @param {(string|Object)} promptItem - The prompt item (string or {name, prompt} object).
+   * @param {(string|Object)} promptItem - The prompt item (string or {name, prompt, files} object).
    * @param {string} systemPrompt - The system prompt to prepend.
    * @param {string} [defaultName] - Default name to use if promptItem is a string or if the object doesn't have a name.
    * @returns {Prompt} A new Prompt object.
    */
   _createPromptObject(promptItem, systemPrompt, defaultName = null) {
-    // Handle both old format (strings) and new format (objects with name and prompt)
+    // Handle both old format (strings) and new format (objects with name, prompt, and files)
     const promptText = typeof promptItem === 'string' ? promptItem : promptItem.prompt;
     const promptName = typeof promptItem === 'string' ? defaultName : (promptItem.name || defaultName);
+    const promptFiles = typeof promptItem === 'string' ? [] : (promptItem.files || []);
+    const cortexPathwayName = typeof promptItem === 'string' ? null : (promptItem.cortexPathwayName || null);
     
-    const messages = [
-      // Add the original prompt as a user message
-      { "role": "user", "content": `{{text}}\n\n${promptText}` },
-    ];
+    const messages = [];
 
     // Only include system message if systemPrompt has content
     if (systemPrompt && systemPrompt.trim() !== "") {
       messages.unshift({ "role": "system", "content": systemPrompt });
     }
 
-    return new Prompt({
+    // If there are files, include chatHistory to get the file content, then add user message
+    if (promptFiles.length > 0) {
+      // Add chatHistory which will contain the resolved file content
+      messages.push("{{chatHistory}}");
+      // Add the user text and prompt as a separate user message
+      messages.push({ "role": "user", "content": `{{text}}\n\n${promptText}` });
+    } else {
+      // No files, just add the user message with text and prompt
+      messages.push({ "role": "user", "content": `{{text}}\n\n${promptText}` });
+    }
+
+    const prompt = new Prompt({
       name: promptName,
       messages: messages
     });
+
+    // Store file hashes for later resolution
+    if (promptFiles.length > 0) {
+      prompt.fileHashes = promptFiles;
+    }
+
+    // Preserve cortexPathwayName if present
+    if (cortexPathwayName) {
+      prompt.cortexPathwayName = cortexPathwayName;
+    }
+
+    return prompt;
   }
 
   /**
    * Transforms the prompts in a pathway to include the system prompt.
    * @param {Object} pathway - The pathway object to transform.
-   * @param {(string[]|Object[])} pathway.prompt - Array of user prompts (strings) or prompt objects with {name, prompt} properties.
+   * @param {(string[]|Object[])} pathway.prompt - Array of user prompts (strings) or prompt objects with {name, prompt, files} properties.
    * @param {string} pathway.systemPrompt - The system prompt to prepend to each user prompt.
    * @returns {Object} A new pathway object with transformed prompts.
    */
@@ -379,6 +401,16 @@ class PathwayManager {
     // Transform each prompt in the array
     newPathway.prompt = prompt.map(p => this._createPromptObject(p, systemPrompt));
 
+    // Collect all file hashes from all prompts
+    const allFileHashes = newPathway.prompt
+      .filter(p => p.fileHashes && p.fileHashes.length > 0)
+      .flatMap(p => p.fileHashes);
+
+    // Store file hashes at pathway level for later resolution
+    if (allFileHashes.length > 0) {
+      newPathway.fileHashes = [...new Set(allFileHashes)]; // Remove duplicates
+    }
+
     return newPathway;
   }
 
@@ -426,6 +458,8 @@ class PathwayManager {
     input PromptInput {
       name: String!
       prompt: String!
+      files: [String!]
+      cortexPathwayName: String
     }
     
     input PathwayInput {

package/lib/pathwayTools.js

@@ -5,6 +5,7 @@ import { publishRequestProgress } from "../lib/redisSubscription.js";
 import { getSemanticChunks } from "../server/chunker.js";
 import logger from '../lib/logger.js';
 import { requestState } from '../server/requestState.js';
+import { processPathwayParameters } from '../server/typeDef.js';
 
 // callPathway - call a pathway from another pathway
 const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
@@ -17,12 +18,18 @@ const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
         throw new Error(`Pathway ${pathwayName} not found`);
     }
 
+    // Merge pathway default parameters with input args, similar to GraphQL typeDef behavior
+    const mergedParams = { ...pathway.defaultInputParameters, ...pathway.inputParameters, ...args };
+    
+    // Process the merged parameters to convert type specification objects to actual values
+    const processedArgs = processPathwayParameters(mergedParams);
+
     const parent = {};
     let rootRequestId = pathwayResolver?.rootRequestId || pathwayResolver?.requestId;
     
     const contextValue = { config, pathway, requestState };
 
-    let data = await pathway.rootResolver(parent, {...args, rootRequestId}, contextValue );
+    let data = await pathway.rootResolver(parent, {...processedArgs, rootRequestId}, contextValue );
 
     if (pathwayResolver && contextValue.pathwayResolver) {
         pathwayResolver.mergeResolver(contextValue.pathwayResolver);

package/lib/redisSubscription.js

@@ -24,6 +24,12 @@ if (connectionString) {
     logger.info(`Using Redis publish for channel(s) ${requestProgressChannel}, ${requestProgressSubscriptionsChannel}`);
     try {
         publisherClient = connectionString && new Redis(connectionString);
+        // Handle Redis publisher client errors to prevent crashes
+        if (publisherClient) {
+            publisherClient.on('error', (error) => {
+                logger.error(`Redis publisherClient error: ${error}`);
+            });
+        }
     } catch (error) {
         logger.error(`Redis connection error: ${error}`);
     }

package/lib/requestExecutor.js

@@ -20,6 +20,10 @@ let client;
 if (connectionString) {
     try {
         client = new Redis(connectionString);
+        // Handle Redis connection errors to prevent crashes
+        client.on('error', (error) => {
+            logger.error(`Redis client connection error: ${error}`);
+        });
     } catch (error) {
         logger.error(`Redis connection error: ${error}`);
     }

package/lib/util.js

@@ -10,6 +10,8 @@ import { promisify } from 'util';
 import { axios } from './requestExecutor.js';
 import { config } from '../config.js';
 import fs from 'fs';
+import path from 'path';
+import FormData from 'form-data';
 
 const pipeline = promisify(stream.pipeline);
 const MEDIA_API_URL = config.get('whisperMediaApiUrl');
@@ -412,6 +414,146 @@ function getAvailableFiles(chatHistory) {
     return availableFiles;
 }
 
+// Helper function to upload base64 image data to cloud storage
+const uploadImageToCloud = async (base64Data, mimeType, pathwayResolver = null) => {
+    let tempFilePath = null;
+    let tempDir = null;
+    
+    try {
+        // Convert base64 to buffer
+        const imageBuffer = Buffer.from(base64Data, 'base64');
+        
+        // Determine file extension from mime type
+        const extension = mimeType.split('/')[1] || 'png';
+        const filename = `generated_image_${Date.now()}.${extension}`;
+        
+        // Create temporary file
+        tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'image-upload-'));
+        tempFilePath = path.join(tempDir, filename);
+        
+        // Write buffer to temp file
+        fs.writeFileSync(tempFilePath, imageBuffer);
+        
+        // Upload to file handler service
+        const fileHandlerUrl = MEDIA_API_URL;
+        if (!fileHandlerUrl) {
+            throw new Error('WHISPER_MEDIA_API_URL is not set');
+        }
+        const requestId = uuidv4();
+        
+        // Create form data for upload
+        const formData = new FormData();
+        formData.append('file', fs.createReadStream(tempFilePath), {
+            filename: filename,
+            contentType: mimeType
+        });
+        
+        // Upload file
+        const uploadResponse = await axios.post(`${fileHandlerUrl}?requestId=${requestId}`, formData, {
+            headers: {
+                ...formData.getHeaders()
+            },
+            timeout: 30000
+        });
+        
+        if (uploadResponse.data && uploadResponse.data.url) {
+            return uploadResponse.data.url;
+        } else {
+            throw new Error('No URL returned from file handler');
+        }
+        
+    } catch (error) {
+        const errorMessage = `Failed to upload image: ${error.message}`;
+        if (pathwayResolver && pathwayResolver.logError) {
+            pathwayResolver.logError(errorMessage);
+        } else {
+            logger.error(errorMessage);
+        }
+        throw error;
+    } finally {
+        // Clean up temp files
+        if (tempFilePath && fs.existsSync(tempFilePath)) {
+            try {
+                fs.unlinkSync(tempFilePath);
+            } catch (cleanupError) {
+                const warningMessage = `Failed to clean up temp file: ${cleanupError.message}`;
+                if (pathwayResolver && pathwayResolver.logWarning) {
+                    pathwayResolver.logWarning(warningMessage);
+                } else {
+                    logger.warn(warningMessage);
+                }
+            }
+        }
+        if (tempDir && fs.existsSync(tempDir)) {
+            try {
+                fs.rmdirSync(tempDir);
+            } catch (cleanupError) {
+                const warningMessage = `Failed to clean up temp directory: ${cleanupError.message}`;
+                if (pathwayResolver && pathwayResolver.logWarning) {
+                    pathwayResolver.logWarning(warningMessage);
+                } else {
+                    logger.warn(warningMessage);
+                }
+            }
+        }
+    }
+};
+
+/**
+ * Convert file hashes to content format suitable for LLM processing
+ * @param {Array<string>} fileHashes - Array of file hashes to resolve
+ * @param {Object} config - Configuration object with file service endpoints
+ * @returns {Promise<Array<string>>} Array of stringified file content objects
+ */
+async function resolveFileHashesToContent(fileHashes, config) {
+    if (!fileHashes || fileHashes.length === 0) return [];
+
+    const fileContentPromises = fileHashes.map(async (hash) => {
+        try {
+            // Use the existing file handler (cortex-file-handler) to resolve file hashes
+            const fileHandlerUrl = config?.get?.('whisperMediaApiUrl');
+            
+            if (fileHandlerUrl && fileHandlerUrl !== 'null') {
+                // Make request to file handler to get file content by hash
+                const response = await axios.get(fileHandlerUrl, { 
+                    params: { hash: hash, checkHash: true } 
+                });
+                if (response.status === 200) {
+                    const fileData = response.data;
+                    const fileUrl = fileData.shortLivedUrl || fileData.url;
+                    const convertedUrl = fileData.converted?.url;
+                    const convertedGcsUrl = fileData.converted?.gcs;
+                    
+                    return JSON.stringify({
+                        type: "image_url",
+                        url: convertedUrl,
+                        image_url: { url: convertedUrl },
+                        gcs: convertedGcsUrl || fileData.gcs, // Add GCS URL for Gemini models
+                        originalFilename: fileData.filename,
+                        hash: hash
+                    });
+                }
+            }
+            
+            // Fallback: create a placeholder that indicates file resolution is needed
+            return JSON.stringify({
+                type: "file_hash",
+                hash: hash,
+                _cortex_needs_resolution: true
+            });
+        } catch (error) {
+            // Return error indicator
+            return JSON.stringify({
+                type: "file_error",
+                hash: hash,
+                error: error.message
+            });
+        }
+    });
+
+    return Promise.all(fileContentPromises);
+}
+
 export { 
     getUniqueId,
     getSearchResultId,
@@ -421,10 +563,12 @@ export {
     chatArgsHasType,
     deleteTempPath,
     downloadFile,
+    uploadImageToCloud,
     convertSrtToText,
     alignSubtitles,
     getMediaChunks,
     markCompletedForCleanUp,
     removeOldImageAndFileContent,
-    getAvailableFiles
+    getAvailableFiles,
+    resolveFileHashesToContent
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.66",
+  "version": "1.4.0",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/basePathway.js

@@ -3,11 +3,11 @@ import { typeDef } from '../server/typeDef.js';
 
 // all default definitions of a single pathway
 export default {
-    prompt: `{{text}}`,
+    prompt: '{{text}}',
     defaultInputParameters: {
-        text: ``,
+        text: '',
         async: false, // switch to enable async mode
-        contextId: ``, // used to identify the context of the request,
+        contextId: '', // used to identify the context of the request,
         stream: false, // switch to enable stream mode
     },
     inputParameters: {},

package/pathways/bing_afagent.js

@@ -5,6 +5,7 @@ import { config } from '../config.js';
 import logger from '../lib/logger.js';
 
 export default {
+    prompt: ["{{text}}"],
     inputParameters: {
         text: ``,
         tool_choice: 'auto',

package/pathways/gemini_15_vision.js

@@ -10,7 +10,7 @@ export default {
         ],
     inputParameters: {
         chatHistory: [{role: '', content: []}],
-        contextId: ``,
+        contextId: '',
     },
     max_tokens: 2048,
     model: 'gemini-pro-15-vision',

package/pathways/google_cse.js

@@ -3,8 +3,8 @@
 
 export default {
     inputParameters: {
-        text: ``,
-        q: ``,
+        text: '',
+        q: '',
         num: 10,
         start: 1,
         safe: 'off', // 'off' | 'active'

package/pathways/image_gemini_25.js

@@ -0,0 +1,85 @@
+import { Prompt } from '../server/prompt.js';
+import { callPathway } from '../lib/pathwayTools.js';
+import logger from '../lib/logger.js';
+
+export default {
+    prompt: [],    
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        let finalPrompt = args.text || '';
+        
+        const { optimizePrompt, input_image, input_image_2, input_image_3 } = { ...resolver.pathway.inputParameters, ...args };
+
+        // Check if prompt optimization is enabled
+        if (optimizePrompt && optimizePrompt !== false && finalPrompt) {
+            try {
+                // Call the prompt optimizer pathway
+                const optimizerResult = await callPathway('image_prompt_optimizer_gemini_25', {
+                    userPrompt: finalPrompt,
+                    hasInputImages: !!input_image || !!input_image_2 || !!input_image_3
+                }, resolver);
+                
+                if (optimizerResult) {
+                    finalPrompt = optimizerResult;
+                }
+            } catch (error) {
+                logger.warn(`Prompt optimization failed, proceeding with original prompt: ${error.message}`);
+            }
+        }
+
+        // Build the user content with text and images
+        const userContent = [{"type": "text", "text": finalPrompt}];
+        
+        // Add input images if provided
+        if (input_image) {
+            userContent.push({
+                "type": "image_url",
+                "image_url": {"url": input_image}
+            });
+        }
+        if (input_image_2) {
+            userContent.push({
+                "type": "image_url", 
+                "image_url": {"url": input_image_2}
+            });
+        }
+        if (input_image_3) {
+            userContent.push({
+                "type": "image_url",
+                "image_url": {"url": input_image_3}
+            });
+        }
+
+        const userMessage = {"role": "user", "content": userContent};
+
+        const systemMessage = {"role": "system", "content": "Instructions:\nYou are Jarvis Vision 2.5, an AI entity working for a prestigious international news agency. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Your primary expertise is both image analysis and image generation/editing. You are capable of:\n\n1. Understanding and interpreting complex image data, identifying patterns and trends\n2. Generating new images based on detailed descriptions\n3. Editing existing images according to specific instructions\n4. Delivering insights and results in a clear, digestible format\n\nYou know the current date and time - it is {{now}}. When generating or editing images, ensure they are appropriate for professional news media use and follow ethical guidelines."};
+
+        const promptMessages = [systemMessage, userMessage];
+
+        resolver.pathwayPrompt = [
+            new Prompt({ messages: promptMessages }),
+        ];
+
+        return await runAllPrompts({ ...args });
+    },
+    
+    inputParameters: {
+        text: "",
+        input_image: "", // URL to first input image
+        input_image_2: "", // URL to second input image  
+        input_image_3: "", // URL to third input image
+        contextId: ``,
+        response_modalities: ["TEXT", "IMAGE"],
+        optimizePrompt: false, // Enable prompt optimization using Google's best practices
+    },
+    max_tokens: 32000,
+    model: 'gemini-25-flash-image',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    geminiSafetySettings: [
+        {category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
+        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}
+    ],
+}