@aj-archipelago/cortex 1.4.6 → 1.4.7

package/pathways/system/entity/tools/sys_tool_image_gemini.js

@@ -1,7 +1,7 @@
 // sys_tool_image_gemini.js
 // Entity tool that creates and modifies images for the entity to show to the user
 import { callPathway } from '../../../../lib/pathwayTools.js';
-import { uploadImageToCloud } from '../../../../lib/util.js';
+import { uploadImageToCloud, addFileToCollection, resolveFileParameter } from '../../../../lib/fileUtils.js';
 
 export default {
     prompt: [],
@@ -9,6 +9,8 @@ export default {
     enableDuplicateRequests: false,
     inputParameters: {
         model: 'oai-gpt4o',
+        contextId: '',
+        contextKey: '',
     },
     timeout: 300,
     toolDefinition: [{
@@ -25,6 +27,17 @@ export default {
                         type: "string",
                         description: "A very detailed prompt describing the image you want to create. You should be very specific - explaining subject matter, style, and details about the image including things like camera angle, lens types, lighting, photographic techniques, etc. Any details you can provide to the image creation engine will help it create the most accurate and useful images. The more detailed and descriptive the prompt, the better the result."
                     },
+                    filenamePrefix: {
+                        type: "string",
+                        description: "Optional: A descriptive prefix to use for the generated image filename (e.g., 'portrait', 'landscape', 'logo'). If not provided, defaults to 'generated-image'."
+                    },
+                    tags: {
+                        type: "array",
+                        items: {
+                            type: "string"
+                        },
+                        description: "Optional: Array of tags to categorize the image (e.g., ['portrait', 'art', 'photography']). Will be merged with default tags ['image', 'generated']."
+                    },
                     userMessage: {
                         type: "string",
                         description: "A user-friendly message that describes what you're doing with this tool"
@@ -46,20 +59,31 @@ export default {
                 properties: {
                     inputImage: {
                         type: "string",
-                        description: "The first image URL copied exactly from an image_url field in your chat context."
+                        description: "An image from your available files (from Available Files section or ListFileCollection or SearchFileCollection) to use as a reference for the image modification."
                     },
                     inputImage2: {
                         type: "string",
-                        description: "The second input image URL copied exactly from an image_url field in your chat context if there is one."
+                        description: "A second image from your available files (from Available Files section or ListFileCollection or SearchFileCollection) to use as a reference for the image modification if there is one."
                     },
                     inputImage3: {
                         type: "string",
-                        description: "The third input image URL copied exactly from an image_url field in your chat context if there is one."
+                        description: "A third image from your available files (from Available Files section or ListFileCollection or SearchFileCollection) to use as a reference for the image modification if there is one."
                     },
                     detailedInstructions: {
                         type: "string",
                         description: "A very detailed prompt describing how you want to modify the image. Be specific about the changes you want to make, including style changes, artistic effects, or specific modifications. The more detailed and descriptive the prompt, the better the result."
                     },
+                    filenamePrefix: {
+                        type: "string",
+                        description: "Optional: A prefix to use for the modified image filename (e.g., 'edited', 'stylized', 'enhanced'). If not provided, defaults to 'modified-image'."
+                    },
+                    tags: {
+                        type: "array",
+                        items: {
+                            type: "string"
+                        },
+                        description: "Optional: Array of tags to categorize the image (e.g., ['edited', 'art', 'stylized']). Will be merged with default tags ['image', 'modified']."
+                    },
                     userMessage: {
                         type: "string",
                         description: "A user-friendly message that describes what you're doing with this tool"
@@ -76,15 +100,55 @@ export default {
             let model = "gemini-25-flash-image";
             let prompt = args.detailedInstructions || "";
             
+            // Resolve input images to URLs using the common utility
+            // For Gemini, prefer GCS URLs over Azure URLs
+            // Fail early if any provided image parameter cannot be resolved
+            if (args.inputImage) {
+                if (!args.contextId) {
+                    throw new Error("contextId is required when using the 'inputImage' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                }
+                const resolved = await resolveFileParameter(args.inputImage, args.contextId, args.contextKey, { preferGcs: true });
+                if (!resolved) {
+                    throw new Error(`File not found: "${args.inputImage}". Use ListFileCollection or SearchFileCollection to find available files.`);
+                }
+                args.inputImage = resolved;
+            }
+            
+            if (args.inputImage2) {
+                if (!args.contextId) {
+                    throw new Error("contextId is required when using the 'inputImage2' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                }
+                const resolved = await resolveFileParameter(args.inputImage2, args.contextId, args.contextKey, { preferGcs: true });
+                if (!resolved) {
+                    throw new Error(`File not found: "${args.inputImage2}". Use ListFileCollection or SearchFileCollection to find available files.`);
+                }
+                args.inputImage2 = resolved;
+            }
+            
+            if (args.inputImage3) {
+                if (!args.contextId) {
+                    throw new Error("contextId is required when using the 'inputImage3' parameter. Use ListFileCollection or SearchFileCollection to find available files.");
+                }
+                const resolved = await resolveFileParameter(args.inputImage3, args.contextId, args.contextKey, { preferGcs: true });
+                if (!resolved) {
+                    throw new Error(`File not found: "${args.inputImage3}". Use ListFileCollection or SearchFileCollection to find available files.`);
+                }
+                args.inputImage3 = resolved;
+            }
+            
+            const resolvedInputImage = args.inputImage;
+            const resolvedInputImage2 = args.inputImage2;
+            const resolvedInputImage3 = args.inputImage3;
+            
             // Call the image generation pathway
             let result = await callPathway('image_gemini_25', {
                 ...args, 
                 text: prompt,
                 model, 
                 stream: false,
-                input_image: args.inputImage,
-                input_image_2: args.inputImage2,
-                input_image_3: args.inputImage3,
+                input_image: resolvedInputImage,
+                input_image_2: resolvedInputImage2,
+                input_image_3: resolvedInputImage3,
                 optimizePrompt: true,
             }, pathwayResolver);
 
@@ -98,13 +162,62 @@ export default {
                     for (const artifact of pathwayResolver.pathwayResultData.artifacts) {
                         if (artifact.type === 'image' && artifact.data && artifact.mimeType) {
                             try {
-                                // Upload image to cloud storage
-                                const imageUrl = await uploadImageToCloud(artifact.data, artifact.mimeType, pathwayResolver);
+                                // Upload image to cloud storage (returns {url, gcs, hash})
+                                const uploadResult = await uploadImageToCloud(artifact.data, artifact.mimeType, pathwayResolver);
+                                
+                                const imageUrl = uploadResult.url || uploadResult;
+                                const imageGcs = uploadResult.gcs || null;
+                                const imageHash = uploadResult.hash || null;
+                                
                                 uploadedImages.push({
                                     type: 'image',
                                     url: imageUrl,
+                                    gcs: imageGcs,
+                                    hash: imageHash,
                                     mimeType: artifact.mimeType
                                 });
+                                
+                                // Add uploaded image to file collection if contextId is available
+                                if (args.contextId && imageUrl) {
+                                    try {
+                                        // Generate filename from mimeType (e.g., "image/png" -> "png")
+                                        const extension = artifact.mimeType.split('/')[1] || 'png';
+                                        // Use hash for uniqueness if available, otherwise use timestamp and index
+                                        const uniqueId = imageHash ? imageHash.substring(0, 8) : `${Date.now()}-${uploadedImages.length}`;
+                                        
+                                        // Determine filename prefix based on whether this is a modification or generation
+                                        // If inputImage exists, it's a modification; otherwise it's a generation
+                                        const isModification = args.inputImage || args.inputImage2 || args.inputImage3;
+                                        const defaultPrefix = isModification ? 'modified-image' : 'generated-image';
+                                        const filenamePrefix = args.filenamePrefix || defaultPrefix;
+                                        
+                                        // Sanitize the prefix to ensure it's a valid filename component
+                                        const sanitizedPrefix = filenamePrefix.replace(/[^a-zA-Z0-9_-]/g, '-').toLowerCase();
+                                        const filename = `${sanitizedPrefix}-${uniqueId}.${extension}`;
+                                        
+                                        // Merge provided tags with default tags
+                                        const defaultTags = ['image', isModification ? 'modified' : 'generated'];
+                                        const providedTags = Array.isArray(args.tags) ? args.tags : [];
+                                        const allTags = [...defaultTags, ...providedTags.filter(tag => !defaultTags.includes(tag))];
+                                        
+                                        // Use the centralized utility function to add to collection
+                                        await addFileToCollection(
+                                            args.contextId,
+                                            args.contextKey || '',
+                                            imageUrl,
+                                            imageGcs,
+                                            filename,
+                                            allTags,
+                                            isModification 
+                                                ? `Modified image from prompt: ${args.detailedInstructions || 'image modification'}`
+                                                : `Generated image from prompt: ${args.detailedInstructions || 'image generation'}`,
+                                            imageHash
+                                        );
+                                    } catch (collectionError) {
+                                        // Log but don't fail - file collection is optional
+                                        pathwayResolver.logWarning(`Failed to add image to file collection: ${collectionError.message}`);
+                                    }
+                                }
                             } catch (uploadError) {
                                 pathwayResolver.logError(`Failed to upload artifact: ${uploadError.message}`);
                                 // Keep original artifact as fallback
@@ -117,7 +230,7 @@ export default {
                     }
                     
                     // Return the urls of the uploaded images as text in the result
-                    result = result + '\n' + uploadedImages.map(image => image.url).join('\n');
+                    result = result ? result + '\n' + uploadedImages.map(image => image.url || image).join('\n') : uploadedImages.map(image => image.url || image).join('\n');
                 }
             } else {
                 // If result is not a CortexResponse, log a warning but return as-is

package/pathways/system/entity/tools/sys_tool_readfile.js

@@ -1,148 +1,241 @@
 // sys_tool_readfile.js
-// Entity tool that reads one or more files and answers questions about them
-
-import { Prompt } from '../../../../server/prompt.js';
+// Tool pathway that reads text files with line number support
+import logger from '../../../../lib/logger.js';
+import { axios } from '../../../../lib/requestExecutor.js';
+import { resolveFileParameter } from '../../../../lib/fileUtils.js';
 
 export default {
-    prompt:
-        [
-            new Prompt({ messages: [
-                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that can view, hear, and understand files of all sorts (images, videos, audio, pdfs, text, etc.) - you provide the capability to view and analyze files that the user provides.\nThe user has provided you with one or more files in this conversation - you should consider them for context when you respond.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.\n{{renderTemplate AI_DATETIME}}`},
-                "{{chatHistory}}",
-            ]}),
-        ],
-    inputParameters: {
-        chatHistory: [{role: '', content: []}],
-        contextId: ``,
-        aiName: "Jarvis",
-        language: "English",
-    },
-    max_tokens: 8192,
-    model: 'gemini-flash-20-vision',
-    useInputChunking: false,
-    enableDuplicateRequests: false,
-    timeout: 600,
-    geminiSafetySettings: [{category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH'},
-        {category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH'},
-        {category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH'},
-        {category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH'}],
-    toolDefinition: [{
+    prompt: [],
+    timeout: 60,
+    toolDefinition: { 
         type: "function",
-        icon: "📄",
+        icon: "📖",
         function: {
-            name: "AnalyzePDF",
-            description: "Use specifically for reading, analyzing, and answering questions about PDF file content. Do not use this tool for analyzing and answering questions about other file types.",
+            name: "ReadFile",
+            description: "Read text content from a file. Can read the entire file or specific line ranges. Use this to access and analyze text files from your file collection. Supports text files, markdown files, html, csv, and other document formats that can be converted to text, but not images, videos, or audio files or pdfs.",
             parameters: {
                 type: "object",
                 properties: {
-                    detailedInstructions: {
+                    file: {
                         type: "string",
-                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
+                        description: "The file to read: can be the file ID, filename, URL, or hash from your file collection. You can find available files in the Available Files section or ListFileCollection or SearchFileCollection."
+                    },
+                    startLine: {
+                        type: "number",
+                        description: "Optional: Starting line number (1-indexed). If not provided, reads from the beginning."
+                    },
+                    endLine: {
+                        type: "number",
+                        description: "Optional: Ending line number (1-indexed). If not provided, reads to the end. Must be >= startLine if startLine is provided."
+                    },
+                    maxLines: {
+                        type: "number",
+                        description: "Optional: Maximum number of lines to read (default: 1000). Use this to limit the size of the response."
                     },
                     userMessage: {
                         type: "string",
                         description: "A user-friendly message that describes what you're doing with this tool"
                     }
                 },
-                required: ["detailedInstructions", "userMessage"]
+                required: ["userMessage"]
             }
         }
     },
-    {
-        type: "function",
-        icon: "📝",
-        function: {
-            name: "AnalyzeText",
-            description: "Use specifically for reading, analyzing, and answering questions about text files (including csv, json, html, etc.).",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
-                        },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                        }
-                },
-                required: ["detailedInstructions", "userMessage"]
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        try {
+            let { cloudUrl, file, startLine, endLine, maxLines = 1000, contextId, contextKey } = args;
+            
+            // If file parameter is provided, resolve it to a URL using the common utility
+            if (file) {
+                if (!contextId) {
+                    const errorResult = {
+                        success: false,
+                        error: "contextId is required when using the 'file' parameter. Use ListFileCollection or SearchFileCollection to find available files."
+                    };
+                    resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                    return JSON.stringify(errorResult);
+                }
+                const resolvedUrl = await resolveFileParameter(file, contextId, contextKey);
+                if (!resolvedUrl) {
+                    const errorResult = {
+                        success: false,
+                        error: `File not found: "${file}". Use ListFileCollection or SearchFileCollection to find available files.`
+                    };
+                    resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                    return JSON.stringify(errorResult);
+                }
+                cloudUrl = resolvedUrl;
             }
-        }
-    },
-    {
-        type: "function",
-        icon: "📝",
-        function: {
-            name: "AnalyzeMarkdown",
-            description: "Use specifically for reading, analyzing, and answering questions about markdown files.",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
-                        },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                        }
-                },
-                required: ["detailedInstructions", "userMessage"]
+            
+            if (!cloudUrl || typeof cloudUrl !== 'string') {
+                const errorResult = {
+                    success: false,
+                    error: "Either cloudUrl or file parameter is required and must be a string"
+                };
+                resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                return JSON.stringify(errorResult);
             }
-        }
-    },   
-    {
-        type: "function",
-        icon: "🖼️",
-        function: {
-            name: "AnalyzeImage",
-            description: "Use specifically for reading, analyzing, and answering questions about image files (jpg, gif, bmp, png, etc). This cannot be used for creating or transforming images.",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
-                    },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                    }
-                },
-                required: ["detailedInstructions", "userMessage"]
+
+            if (startLine !== undefined) {
+                if (typeof startLine !== 'number' || !Number.isInteger(startLine) || startLine < 1) {
+                const errorResult = {
+                    success: false,
+                        error: "startLine must be a positive integer (1-indexed)"
+                };
+                resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                return JSON.stringify(errorResult);
+                }
             }
-        }
-    },
-    {
-        type: "function",
-        icon: "🎥",
-        function: {
-            name: "AnalyzeVideo",
-            description: "Use specifically for reading, analyzing, and answering questions about video or audio file content. You MUST use this tool to look at video or audio files.",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed instructions about what you need the tool to do - questions you need answered about the files, etc."
-                    },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                    }
-                },
-                required: ["detailedInstructions", "userMessage"]
+
+            if (endLine !== undefined) {
+                if (typeof endLine !== 'number' || !Number.isInteger(endLine) || endLine < 1) {
+                const errorResult = {
+                    success: false,
+                        error: "endLine must be a positive integer (1-indexed)"
+                };
+                resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                return JSON.stringify(errorResult);
+                }
             }
+
+            if (startLine !== undefined && endLine !== undefined && endLine < startLine) {
+                const errorResult = {
+                    success: false,
+                    error: "endLine must be >= startLine"
+                };
+                resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                return JSON.stringify(errorResult);
+            }
+
+            if (maxLines !== undefined) {
+                if (typeof maxLines !== 'number' || !Number.isInteger(maxLines) || maxLines < 1) {
+                    const errorResult = {
+                        success: false,
+                        error: "maxLines must be a positive integer"
+                    };
+                    resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                    return JSON.stringify(errorResult);
+                }
+            }
+            // Download file content directly from the URL (don't use file handler for content)
+            // Use arraybuffer and explicitly decode as UTF-8 to avoid encoding issues
+            const response = await axios.get(cloudUrl, {
+                responseType: 'arraybuffer',
+                timeout: 30000,
+                validateStatus: (status) => status >= 200 && status < 400
+            });
+
+            if (response.status !== 200 || !response.data) {
+                throw new Error(`Failed to download file content: ${response.status}`);
+            }
+
+            // Explicitly decode as UTF-8 to prevent mojibake (encoding corruption)
+            const textContent = Buffer.from(response.data).toString('utf8');
+            const allLines = textContent.split(/\r?\n/);
+            const totalLines = allLines.length;
+
+            // Handle empty file
+            if (totalLines === 0 || (totalLines === 1 && allLines[0] === '')) {
+                const result = {
+                    success: true,
+                    cloudUrl: cloudUrl,
+                    totalLines: 0,
+                    returnedLines: 0,
+                    startLine: 1,
+                    endLine: 0,
+                    content: '',
+                    truncated: false,
+                    isEmpty: true
+                };
+                resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                return JSON.stringify(result);
+            }
+
+            // Apply line range filtering
+            let selectedLines = allLines;
+            let actualStartLine = 1;
+            let actualEndLine = totalLines;
+            let wasTruncatedByRange = false;
+
+            if (startLine !== undefined || endLine !== undefined) {
+                const start = startLine !== undefined ? Math.max(1, Math.min(startLine, totalLines)) - 1 : 0; // Convert to 0-indexed, clamp to valid range
+                const end = endLine !== undefined ? Math.min(totalLines, Math.max(1, endLine)) : totalLines; // Clamp to valid range
+                
+                if (startLine !== undefined && startLine > totalLines) {
+                    const errorResult = {
+                        success: false,
+                        error: `startLine (${startLine}) exceeds file length (${totalLines} lines)`
+                    };
+                    resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+                    return JSON.stringify(errorResult);
+                }
+
+                selectedLines = allLines.slice(start, end);
+                actualStartLine = start + 1; // Convert back to 1-indexed
+                actualEndLine = end;
+                wasTruncatedByRange = (endLine !== undefined && endLine < totalLines) || (startLine !== undefined && startLine > 1);
+            }
+
+            // Apply maxLines limit
+            let wasTruncatedByMaxLines = false;
+            if (selectedLines.length > maxLines) {
+                selectedLines = selectedLines.slice(0, maxLines);
+                wasTruncatedByMaxLines = true;
+            }
+
+            const result = {
+                success: true,
+                cloudUrl: cloudUrl,
+                totalLines: totalLines,
+                returnedLines: selectedLines.length,
+                startLine: actualStartLine,
+                endLine: actualEndLine,
+                content: selectedLines.join('\n'),
+                truncated: wasTruncatedByRange || wasTruncatedByMaxLines,
+                truncatedByRange: wasTruncatedByRange,
+                truncatedByMaxLines: wasTruncatedByMaxLines
+            };
+
+            resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+            return JSON.stringify(result);
+
+        } catch (e) {
+            let errorMsg;
+            if (e?.message) {
+                errorMsg = e.message;
+            } else if (e?.response) {
+                // Handle HTTP errors
+                const status = e.response.status;
+                const statusText = e.response.statusText || '';
+                errorMsg = `HTTP ${status}${statusText ? ` ${statusText}` : ''}: Failed to download file`;
+            } else if (e?.code === 'ECONNABORTED' || e?.code === 'ETIMEDOUT') {
+                errorMsg = 'Request timeout: File download took too long';
+            } else if (e?.code === 'ENOTFOUND' || e?.code === 'ECONNREFUSED') {
+                errorMsg = `Connection error: ${e.message || 'Unable to reach file server'}`;
+            } else if (typeof e === 'string') {
+                errorMsg = e;
+            } else if (e?.errors && Array.isArray(e.errors)) {
+                // Handle AggregateError
+                errorMsg = e.errors.map(err => err?.message || String(err)).join('; ');
+            } else if (e) {
+                errorMsg = String(e);
+            } else {
+                errorMsg = 'Unknown error occurred while reading file';
+            }
+
+            logger.error(`Error reading cloud file ${cloudUrl || file || 'unknown'}: ${errorMsg}`);
+            
+            const errorResult = {
+                success: false,
+                cloudUrl: cloudUrl || null,
+                file: file || null,
+                error: errorMsg
+            };
+
+            resolver.tool = JSON.stringify({ toolUsed: "ReadFile" });
+            return JSON.stringify(errorResult);
         }
-    }],
-    
-    executePathway: async ({args, runAllPrompts, resolver}) => {
-        if (args.detailedInstructions) {
-            args.chatHistory.push({role: "user", content: args.detailedInstructions});
-        }
-        const result = await runAllPrompts({ ...args });
-        resolver.tool = JSON.stringify({ toolUsed: "vision" });
-        return result;
     }
-}
+};
+