@aj-archipelago/cortex 1.4.26 → 1.4.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.4.26",
+  "version": "1.4.27",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/tools/sys_tool_analyzefile.js

@@ -163,7 +163,23 @@ export default {
             
             // Generate file message content if provided
             if (args.file) {
-                const fileContent = await generateFileMessageContent(args.file, args.contextId, args.contextKey);
+                // Use agentContext if available, otherwise fall back to creating it from contextId/contextKey
+                const agentContext = args.agentContext || (args.contextId ? [{
+                    contextId: args.contextId,
+                    contextKey: args.contextKey || null,
+                    default: true
+                }] : null);
+                
+                if (!agentContext || !Array.isArray(agentContext) || agentContext.length === 0) {
+                    const errorMessage = `File not found: "${args.file}". agentContext is required to look up files in the collection.`;
+                    resolver.tool = JSON.stringify({ toolUsed: "vision" });
+                    return JSON.stringify({ 
+                        error: errorMessage,
+                        recoveryMessage: "The file was not found. Please verify the file exists in the collection or provide a valid file reference."
+                    });
+                }
+                
+                const fileContent = await generateFileMessageContent(args.file, agentContext);
                 if (!fileContent) {
                     const errorMessage = `File not found: "${args.file}". Use ListFileCollection or SearchFileCollection to find available files.`;
                     resolver.tool = JSON.stringify({ toolUsed: "vision" });

package/pathways/system/entity/tools/sys_tool_image.js

@@ -88,6 +88,7 @@ export default {
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
         const pathwayResolver = resolver;
+        const chatId = args.chatId || null;
 
         try {   
             let model = "replicate-seedream-4";
@@ -104,8 +105,8 @@ export default {
             // Fail early if any provided image cannot be resolved
             const resolvedInputImages = [];
             if (args.inputImages && Array.isArray(args.inputImages)) {
-                if (!args.contextId) {
-                    throw new Error("contextId is required when using the 'inputImages' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                if (!args.agentContext || !Array.isArray(args.agentContext) || args.agentContext.length === 0) {
+                    throw new Error("agentContext is required when using the 'inputImages' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
                 }
                 
                 // Limit to 3 images maximum
@@ -175,13 +176,13 @@ export default {
                                 const uploadedGcs = uploadResult.gcs || null;
                                 const uploadedHash = uploadResult.hash || null;
                                 
-                                uploadedImages.push({
+                                const imageData = {
                                     type: 'image',
                                     url: uploadedUrl,
                                     gcs: uploadedGcs,
                                     hash: uploadedHash,
                                     mimeType: mimeType
-                                });
+                                };
                                 
                                 // Add uploaded image to file collection if contextId is available
                                 if (args.contextId && uploadedUrl) {
@@ -205,8 +206,8 @@ export default {
                                         const providedTags = Array.isArray(args.tags) ? args.tags : [];
                                         const allTags = [...defaultTags, ...providedTags.filter(tag => !defaultTags.includes(tag))];
                                         
-                                        // Use the centralized utility function to add to collection
-                                        await addFileToCollection(
+                                        // Use the centralized utility function to add to collection - capture returned entry
+                                        const fileEntry = await addFileToCollection(
                                             args.contextId,
                                             args.contextKey || '',
                                             uploadedUrl,
@@ -219,13 +220,19 @@ export default {
                                             uploadedHash,
                                             null, // fileUrl - not needed since we already uploaded
                                             pathwayResolver,
-                                            true // permanent => retention=permanent
+                                            true, // permanent => retention=permanent
+                                            chatId
                                         );
+                                        
+                                        // Use the file entry data for the return message
+                                        imageData.fileEntry = fileEntry;
                                     } catch (collectionError) {
                                         // Log but don't fail - file collection is optional
                                         pathwayResolver.logWarning(`Failed to add image to file collection: ${collectionError.message}`);
                                     }
                                 }
+                                
+                                uploadedImages.push(imageData);
                             } catch (uploadError) {
                                 pathwayResolver.logError(`Failed to upload image from Replicate: ${uploadError.message}`);
                                 // Keep original URL as fallback
@@ -241,11 +248,59 @@ export default {
                         }
                     }
                     
-                    // Return the URLs of the uploaded images as text in the result
+                    // Return the URLs of the uploaded images in structured format
                     // Replace the result with uploaded cloud URLs (not the original Replicate URLs)
                     if (uploadedImages.length > 0) {
-                        const imageUrls = uploadedImages.map(image => image.url || image).filter(Boolean);
-                        result = imageUrls.join('\n');
+                        const successfulImages = uploadedImages.filter(img => img.url);
+                        if (successfulImages.length > 0) {
+                            // Build imageUrls array in the format expected by pathwayTools.js for toolImages injection
+                            // This format matches ViewImages tool so images get properly injected into chat history
+                            const imageUrls = successfulImages.map((img) => {
+                                const url = img.fileEntry?.url || img.url;
+                                const gcs = img.fileEntry?.gcs || img.gcs;
+                                const hash = img.fileEntry?.hash || img.hash;
+                                
+                                return {
+                                    type: "image_url",
+                                    url: url,
+                                    gcs: gcs || null,
+                                    image_url: { url: url },
+                                    hash: hash || null
+                                };
+                            });
+                            
+                            // Return image info in the same format as availableFiles for the text message
+                            // Format: hash | filename | url | date | tags
+                            const imageList = successfulImages.map((img) => {
+                                if (img.fileEntry) {
+                                    // Use the file entry data from addFileToCollection
+                                    const fe = img.fileEntry;
+                                    const dateStr = fe.addedDate 
+                                        ? new Date(fe.addedDate).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })
+                                        : '';
+                                    const tagsStr = Array.isArray(fe.tags) ? fe.tags.join(',') : '';
+                                    return `${fe.hash || ''} | ${fe.displayFilename || ''} | ${fe.url || img.url} | ${dateStr} | ${tagsStr}`;
+                                } else {
+                                    // Fallback if file collection wasn't available
+                                    return `${img.hash || 'unknown'} | | ${img.url} | |`;
+                                }
+                            }).join('\n');
+                            
+                            const count = successfulImages.length;
+                            const isModification = args.inputImages && Array.isArray(args.inputImages) && args.inputImages.length > 0;
+                            
+                            // Make the success message very explicit so the agent knows files were created and added to collection
+                            // This format matches availableFiles so the agent can reference them by hash/filename
+                            const action = isModification ? 'Image modification' : 'Image generation';
+                            const message = `${action} completed successfully. ${count} image${count > 1 ? 's have' : ' has'} been generated, uploaded to cloud storage, and added to your file collection. The image${count > 1 ? 's are' : ' is'} now available in your file collection:\n\n${imageList}\n\nYou can reference these images by their hash, filename, or URL in future tool calls.`;
+                            
+                            // Return JSON object with imageUrls (kept for backward compatibility, but explicit message should prevent looping)
+                            result = JSON.stringify({
+                                success: true,
+                                message: message,
+                                imageUrls: imageUrls
+                            });
+                        }
                     }
                 }
             }

package/pathways/system/entity/tools/sys_tool_image_gemini.js

@@ -90,6 +90,7 @@ export default {
     }],
     executePathway: async ({args, runAllPrompts, resolver}) => {
         const pathwayResolver = resolver;
+        const chatId = args.chatId || null;
 
         try {   
             let model = "gemini-flash-25-image";
@@ -100,8 +101,8 @@ export default {
             // Fail early if any provided image cannot be resolved
             const resolvedInputImages = [];
             if (args.inputImages && Array.isArray(args.inputImages)) {
-                if (!args.contextId) {
-                    throw new Error("contextId is required when using the 'inputImages' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                if (!args.agentContext || !Array.isArray(args.agentContext) || args.agentContext.length === 0) {
+                    throw new Error("agentContext is required when using the 'inputImages' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
                 }
                 
                 // Limit to 3 images maximum
@@ -203,7 +204,8 @@ export default {
                                         imageHash,
                                         null,
                                         pathwayResolver,
-                                        true // permanent => retention=permanent
+                                        true, // permanent => retention=permanent
+                                        chatId
                                     );
                                     
                                     // Use the file entry data for the return message
@@ -229,7 +231,23 @@ export default {
                 // Check if we successfully uploaded any images
                 const successfulImages = uploadedImages.filter(img => img.url);
                 if (successfulImages.length > 0) {
-                    // Return image info in the same format as availableFiles
+                    // Build imageUrls array in the format expected by pathwayTools.js for toolImages injection
+                    // This format matches ViewImages tool so images get properly injected into chat history
+                    const imageUrls = successfulImages.map((img) => {
+                        const url = img.fileEntry?.url || img.url;
+                        const gcs = img.fileEntry?.gcs || img.gcs;
+                        const hash = img.fileEntry?.hash || img.hash;
+                        
+                        return {
+                            type: "image_url",
+                            url: url,
+                            gcs: gcs || null,
+                            image_url: { url: url },
+                            hash: hash || null
+                        };
+                    });
+                    
+                    // Return image info in the same format as availableFiles for the text message
                     // Format: hash | filename | url | date | tags
                     const imageList = successfulImages.map((img) => {
                         if (img.fileEntry) {
@@ -247,7 +265,17 @@ export default {
                     }).join('\n');
                     
                     const count = successfulImages.length;
-                    return `Image generation successful. Generated ${count} image${count > 1 ? 's' : ''}:\n${imageList}`;
+                    
+                    // Make the success message very explicit so the agent knows files were created and added to collection
+                    // This format matches availableFiles so the agent can reference them by hash/filename
+                    const message = `Image generation completed successfully. ${count} image${count > 1 ? 's have' : ' has'} been generated, uploaded to cloud storage, and added to your file collection. The image${count > 1 ? 's are' : ' is'} now available in your file collection:\n\n${imageList}\n\nYou can reference these images by their hash, filename, or URL in future tool calls.`;
+                    
+                    // Return JSON object with imageUrls (kept for backward compatibility, but explicit message should prevent looping)
+                    return JSON.stringify({
+                        success: true,
+                        message: message,
+                        imageUrls: imageUrls
+                    });
                 } else {
                     throw new Error('Image generation failed: Images were generated but could not be uploaded to storage');
                 }

package/pathways/system/entity/tools/sys_tool_slides_gemini.js

@@ -71,8 +71,8 @@ export default {
             // Fail early if any provided image cannot be resolved
             const resolvedInputImages = [];
             if (args.inputImages && Array.isArray(args.inputImages)) {
-                if (!args.contextId) {
-                    throw new Error("contextId is required when using the 'inputImages' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                if (!args.agentContext || !Array.isArray(args.agentContext) || args.agentContext.length === 0) {
+                    throw new Error("agentContext is required when using the 'inputImages' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
                 }
                 
                 // Limit to 3 images maximum
@@ -210,7 +210,23 @@ export default {
                 // Check if we successfully uploaded any images
                 const successfulImages = uploadedImages.filter(img => img.url);
                 if (successfulImages.length > 0) {
-                    // Return image info in the same format as availableFiles
+                    // Build imageUrls array in the format expected by pathwayTools.js for toolImages injection
+                    // This format matches ViewImages tool so images get properly injected into chat history
+                    const imageUrls = successfulImages.map((img) => {
+                        const url = img.fileEntry?.url || img.url;
+                        const gcs = img.fileEntry?.gcs || img.gcs;
+                        const hash = img.fileEntry?.hash || img.hash;
+                        
+                        return {
+                            type: "image_url",
+                            url: url,
+                            gcs: gcs || null,
+                            image_url: { url: url },
+                            hash: hash || null
+                        };
+                    });
+                    
+                    // Return image info in the same format as availableFiles for the text message
                     // Format: hash | filename | url | date | tags
                     const imageList = successfulImages.map((img) => {
                         if (img.fileEntry) {
@@ -228,7 +244,18 @@ export default {
                     }).join('\n');
                     
                     const count = successfulImages.length;
-                    return `Slide/infographic generation successful. Generated ${count} image${count > 1 ? 's' : ''}:\n${imageList}`;
+                    
+                    // Make the success message very explicit so the agent knows files were created and added to collection
+                    // This format matches availableFiles so the agent can reference them by hash/filename
+                    const message = `Slide/infographic generation completed successfully. ${count} image${count > 1 ? 's have' : ' has'} been generated, uploaded to cloud storage, and added to your file collection. The image${count > 1 ? 's are' : ' is'} now available in your file collection:\n\n${imageList}\n\nYou can reference these images by their hash, filename, or URL in future tool calls.`;
+                    
+                    // Return JSON object with imageUrls (kept for backward compatibility, but explicit message should prevent looping)
+                    // This prevents the agent from looping because it can't see the generated images
+                    return JSON.stringify({
+                        success: true,
+                        message: message,
+                        imageUrls: imageUrls
+                    });
                 } else {
                     throw new Error('Slide generation failed: Content was generated but could not be uploaded to storage');
                 }

package/pathways/system/entity/tools/sys_tool_video_veo.js

@@ -115,6 +115,7 @@ export default {
     }],
     executePathway: async ({args, runAllPrompts, resolver}) => {
         const pathwayResolver = resolver;
+        const chatId = args.chatId || null;
 
         try {   
             const model = "veo-3.1-fast-generate";
@@ -124,8 +125,8 @@ export default {
             // Veo requires GCS URLs for input images
             let imageParam = undefined;
             if (args.inputImage) {
-                if (!args.contextId) {
-                    throw new Error("contextId is required when using the 'inputImage' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                if (!args.agentContext || !Array.isArray(args.agentContext) || args.agentContext.length === 0) {
+                    throw new Error("agentContext is required when using the 'inputImage' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
                 }
                 
                 const resolved = await resolveFileParameter(args.inputImage, args.agentContext, { preferGcs: true });
@@ -297,7 +298,8 @@ export default {
                                     uploadedHash,
                                     null,
                                     pathwayResolver,
-                                    true // permanent => retention=permanent
+                                    true, // permanent => retention=permanent
+                                    chatId
                                 );
                                 
                                 // Use the file entry data for the return message
@@ -323,11 +325,27 @@ export default {
                 }
             }
             
-            // Return the URLs of the uploaded videos as text in the result
+            // Return the URLs of the uploaded videos in structured format
             if (uploadedVideos.length > 0) {
                 const successfulVideos = uploadedVideos.filter(v => v.url);
                 if (successfulVideos.length > 0) {
-                    // Return video info in the same format as availableFiles
+                    // Build imageUrls array in the format expected by pathwayTools.js for toolImages injection
+                    // Videos can use image_url type since they can be displayed with markdown image syntax
+                    const imageUrls = successfulVideos.map((vid) => {
+                        const url = vid.fileEntry?.url || vid.url;
+                        const gcs = vid.fileEntry?.gcs || vid.gcs;
+                        const hash = vid.fileEntry?.hash || vid.hash;
+                        
+                        return {
+                            type: "image_url",
+                            url: url,
+                            gcs: gcs || null,
+                            image_url: { url: url },
+                            hash: hash || null
+                        };
+                    });
+                    
+                    // Return video info in the same format as availableFiles for the text message
                     // Format: hash | filename | url | date | tags
                     const videoList = successfulVideos.map((vid) => {
                         if (vid.fileEntry) {
@@ -345,8 +363,17 @@ export default {
                     }).join('\n');
                     
                     const count = successfulVideos.length;
-                    // Note: The UI supports displaying videos using markdown image syntax
-                    return `Video generation successful. Generated ${count} video${count > 1 ? 's' : ''}. Videos can be displayed using markdown image syntax, e.g. ![video](url)\n${videoList}`;
+                    
+                    // Make the success message very explicit so the agent knows files were created and added to collection
+                    // This format matches availableFiles so the agent can reference them by hash/filename
+                    const message = `Video generation completed successfully. ${count} video${count > 1 ? 's have' : ' has'} been generated, uploaded to cloud storage, and added to your file collection. The video${count > 1 ? 's are' : ' is'} now available in your file collection:\n\n${videoList}\n\nYou can reference these videos by their hash, filename, or URL in future tool calls. Videos can be displayed using markdown image syntax, e.g. ![video](url)`;
+                    
+                    // Return JSON object with imageUrls (kept for backward compatibility, but explicit message should prevent looping)
+                    return JSON.stringify({
+                        success: true,
+                        message: message,
+                        imageUrls: imageUrls
+                    });
                 } else {
                     // All videos failed to upload
                     const errors = uploadedVideos.map(v => v.error).filter(Boolean);

package/tests/integration/features/tools/fileCollection.test.js

@@ -1996,6 +1996,94 @@ test('Analyzer tool: Returns error JSON format when file not found', async t =>
     }
 });
 
+test('Analyzer tool: Works with legacy contextId/contextKey parameters (backward compatibility)', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // First add a file to the collection
+        await callPathway('sys_tool_file_collection', {
+            agentContext: [{ contextId, contextKey: null, default: true }],
+            url: 'https://example.com/test-document.pdf',
+            filename: 'test-document.pdf',
+            userMessage: 'Add test file for analyzer'
+        });
+        
+        // Get the file ID from the collection
+        const collection = await loadFileCollection(contextId, null, false);
+        const fileId = collection[0].id;
+        
+        // Test analyzer tool with legacy contextId/contextKey (without agentContext)
+        // This tests that the tool correctly handles backward compatibility
+        const result = await callPathway('sys_tool_analyzefile', {
+            contextId,  // Legacy format - no agentContext
+            contextKey: null,
+            file: fileId,
+            detailedInstructions: 'What is this file?',
+            userMessage: 'Testing backward compatibility'
+        });
+        
+        t.truthy(result, 'Should have a result');
+        
+        // The result should be a string (not an error JSON)
+        // If it's an error, it should be properly formatted
+        let parsedResult;
+        try {
+            parsedResult = JSON.parse(result);
+            // If it parsed as JSON, check if it's an error
+            if (parsedResult.error) {
+                t.fail(`Tool returned error when it should have worked: ${parsedResult.error}`);
+            }
+        } catch (error) {
+            // If it doesn't parse as JSON, that's fine - it's likely the model response
+            t.truthy(typeof result === 'string', 'Result should be a string');
+        }
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
+test('Analyzer tool: File resolution works with agentContext', async t => {
+    const contextId = createTestContext();
+    
+    try {
+        // Add a file to the collection
+        await callPathway('sys_tool_file_collection', {
+            agentContext: [{ contextId, contextKey: null, default: true }],
+            url: 'https://example.com/test-file.pdf',
+            filename: 'test-file.pdf',
+            userMessage: 'Add test file'
+        });
+        
+        // Get the file ID from the collection
+        const collection = await loadFileCollection(contextId, null, false);
+        const fileId = collection[0].id;
+        
+        // Test analyzer tool with agentContext (modern format)
+        const result = await callPathway('sys_tool_analyzefile', {
+            agentContext: [{ contextId, contextKey: null, default: true }],
+            file: fileId,
+            detailedInstructions: 'What is this file?',
+            userMessage: 'Testing with agentContext'
+        });
+        
+        t.truthy(result, 'Should have a result');
+        
+        // The result should not be an error
+        let parsedResult;
+        try {
+            parsedResult = JSON.parse(result);
+            if (parsedResult.error) {
+                t.fail(`Tool returned error: ${parsedResult.error}`);
+            }
+        } catch (error) {
+            // If it doesn't parse as JSON, that's fine - it's likely the model response
+            t.truthy(typeof result === 'string', 'Result should be a string');
+        }
+    } finally {
+        await cleanup(contextId);
+    }
+});
+
 // ============================================
 // Converted Files Tests (displayFilename != URL extension)
 // ============================================