@aj-archipelago/cortex 1.3.62 → 1.3.64

package/server/plugins/grokVisionPlugin.js

@@ -0,0 +1,365 @@
+import OpenAIVisionPlugin from './openAiVisionPlugin.js';
+import logger from '../../lib/logger.js';
+import { extractCitationTitle } from '../../lib/util.js';
+import CortexResponse from '../../lib/cortexResponse.js';
+
+export function safeJsonParse(content) {
+    try {
+        const parsedContent = JSON.parse(content);
+        return (typeof parsedContent === 'object' && parsedContent !== null) ? parsedContent : content;
+    } catch (e) {
+        return content;
+    }
+}
+
+class GrokVisionPlugin extends OpenAIVisionPlugin {
+
+    constructor(pathway, model) {
+        super(pathway, model);
+        // Grok is always multimodal, so we inherit all vision capabilities from OpenAIVisionPlugin
+    }
+
+    // Override the logging function to display Grok-specific messages
+    logRequestData(data, responseData, prompt) {
+        const { stream, messages } = data;
+        if (messages && messages.length > 1) {
+            logger.info(`[grok request sent containing ${messages.length} messages]`);
+            let totalLength = 0;
+            let totalUnits;
+            messages.forEach((message, index) => {
+                //message.content string or array
+                const content = message.content === undefined ? JSON.stringify(message) : (Array.isArray(message.content) ? message.content.map(item => {
+                    if (item.type === 'image_url' && item.image_url?.url?.startsWith('data:')) {
+                        return JSON.stringify({
+                            type: 'image_url',
+                            image_url: { url: '* base64 data truncated for log *' }
+                        });
+                    }
+                    return JSON.stringify(item);
+                }).join(', ') : message.content);
+                const { length, units } = this.getLength(content);
+                const displayContent = this.shortenContent(content);
+
+                let logMessage = `message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`;
+                
+                // Add tool calls to log if they exist
+                if (message.role === 'assistant' && message.tool_calls) {
+                    logMessage += `, tool_calls: ${JSON.stringify(message.tool_calls)}`;
+                }
+                
+                logger.verbose(logMessage);
+                totalLength += length;
+                totalUnits = units;
+            });
+            logger.info(`[grok request contained ${totalLength} ${totalUnits}]`);
+        } else {
+            const message = messages[0];
+            const content = Array.isArray(message.content) ? message.content.map(item => {
+                if (item.type === 'image_url' && item.image_url?.url?.startsWith('data:')) {
+                    return JSON.stringify({
+                        type: 'image_url',
+                        image_url: { url: '* base64 data truncated for log *' }
+                    });
+                }
+                return JSON.stringify(item);
+            }).join(', ') : message.content;
+            const { length, units } = this.getLength(content);
+            logger.info(`[grok request sent containing ${length} ${units}]`);
+            logger.verbose(`${this.shortenContent(content)}`);
+        }
+        if (stream) {
+            logger.info(`[grok response received as an SSE stream]`);
+        } else {
+            const parsedResponse = this.parseResponse(responseData);
+            
+            if (typeof parsedResponse === 'string') {
+                const { length, units } = this.getLength(parsedResponse);
+                logger.info(`[grok response received containing ${length} ${units}]`);
+                logger.verbose(`${this.shortenContent(parsedResponse)}`);
+            } else {
+                logger.info(`[grok response received containing object]`);
+                logger.verbose(`${JSON.stringify(parsedResponse)}`);
+            }
+        }
+
+        prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+    }
+
+
+    // Validate live search parameters according to X.AI documentation
+    validateSearchParameters(searchParams) {
+        const errors = [];
+
+        // Validate 'mode' parameter
+        if (searchParams.mode !== undefined) {
+            const validModes = ['off', 'auto', 'on'];
+            if (!validModes.includes(searchParams.mode)) {
+                errors.push(`Invalid 'mode' parameter: ${searchParams.mode}. Must be one of: ${validModes.join(', ')}`);
+            }
+        }
+
+        // Validate 'sources' parameter
+        if (searchParams.sources !== undefined) {
+            if (!Array.isArray(searchParams.sources)) {
+                errors.push("'sources' must be an array");
+            } else {
+                const validSourceTypes = ['web', 'news', 'x', 'rss'];
+                searchParams.sources.forEach((source, index) => {
+                    if (!source || typeof source !== 'object') {
+                        errors.push(`Source at index ${index} must be an object`);
+                        return;
+                    }
+
+                    if (!validSourceTypes.includes(source.type)) {
+                        errors.push(`Invalid source type at index ${index}: ${source.type}. Must be one of: ${validSourceTypes.join(', ')}`);
+                    }
+
+                    // Validate source-specific parameters
+                    if (source.type === 'web' || source.type === 'news') {
+                        if (source.country !== undefined && typeof source.country !== 'string') {
+                            errors.push(`Source at index ${index}: 'country' must be a string`);
+                        }
+                        if (source.excluded_websites !== undefined && !Array.isArray(source.excluded_websites)) {
+                            errors.push(`Source at index ${index}: 'excluded_websites' must be an array`);
+                        }
+                        if (source.allowed_websites !== undefined && !Array.isArray(source.allowed_websites)) {
+                            errors.push(`Source at index ${index}: 'allowed_websites' must be an array`);
+                        }
+                        if (source.safe_search !== undefined && typeof source.safe_search !== 'boolean') {
+                            errors.push(`Source at index ${index}: 'safe_search' must be a boolean`);
+                        }
+                    }
+
+                    if (source.type === 'x') {
+                        if (source.included_x_handles !== undefined && !Array.isArray(source.included_x_handles)) {
+                            errors.push(`Source at index ${index}: 'included_x_handles' must be an array`);
+                        } else if (source.included_x_handles !== undefined && source.included_x_handles.length > 10) {
+                            errors.push(`Source at index ${index}: 'included_x_handles' can have a maximum of 10 items`);
+                        }
+                        
+                        if (source.excluded_x_handles !== undefined && !Array.isArray(source.excluded_x_handles)) {
+                            errors.push(`Source at index ${index}: 'excluded_x_handles' must be an array`);
+                        } else if (source.excluded_x_handles !== undefined && source.excluded_x_handles.length > 10) {
+                            errors.push(`Source at index ${index}: 'excluded_x_handles' can have a maximum of 10 items`);
+                        }
+                        
+                        // Check that both handles arrays are not specified simultaneously
+                        if (source.included_x_handles !== undefined && source.excluded_x_handles !== undefined) {
+                            errors.push(`Source at index ${index}: 'included_x_handles' and 'excluded_x_handles' cannot be specified simultaneously`);
+                        }
+                        
+                        if (source.post_favorite_count !== undefined && typeof source.post_favorite_count !== 'number') {
+                            errors.push(`Source at index ${index}: 'post_favorite_count' must be a number`);
+                        }
+                        if (source.post_view_count !== undefined && typeof source.post_view_count !== 'number') {
+                            errors.push(`Source at index ${index}: 'post_view_count' must be a number`);
+                        }
+                    }
+
+                    if (source.type === 'rss') {
+                        if (source.links !== undefined && !Array.isArray(source.links)) {
+                            errors.push(`Source at index ${index}: 'links' must be an array`);
+                        } else if (source.links !== undefined && source.links.length > 1) {
+                            errors.push(`Source at index ${index}: 'links' can only have one item`);
+                        }
+                    }
+                });
+            }
+        }
+
+        // Validate 'return_citations' parameter
+        if (searchParams.return_citations !== undefined && typeof searchParams.return_citations !== 'boolean') {
+            errors.push("'return_citations' must be a boolean");
+        }
+
+        // Validate date parameters
+        const dateFormat = /^\d{4}-\d{2}-\d{2}$/;
+        ['from_date', 'to_date'].forEach(dateField => {
+            if (searchParams[dateField] !== undefined) {
+                if (typeof searchParams[dateField] !== 'string') {
+                    errors.push(`'${dateField}' must be a string`);
+                } else if (!dateFormat.test(searchParams[dateField])) {
+                    errors.push(`'${dateField}' must be in YYYY-MM-DD format`);
+                } else {
+                    // Validate that the date is actually valid
+                    const date = new Date(searchParams[dateField]);
+                    if (isNaN(date.getTime()) || date.toISOString().split('T')[0] !== searchParams[dateField]) {
+                        errors.push(`'${dateField}' is not a valid date`);
+                    }
+                }
+            }
+        });
+
+        // Validate 'max_search_results' parameter
+        if (searchParams.max_search_results !== undefined) {
+            if (typeof searchParams.max_search_results !== 'number' || !Number.isInteger(searchParams.max_search_results)) {
+                errors.push("'max_search_results' must be an integer");
+            } else if (searchParams.max_search_results <= 0) {
+                errors.push("'max_search_results' must be a positive integer");
+            } else if (searchParams.max_search_results > 50) {
+                errors.push("'max_search_results' must be 50 or less");
+            }
+        }
+
+        if (errors.length > 0) {
+            throw new Error(`Live Search parameter validation failed:\n${errors.join('\n')}`);
+        }
+
+        return true;
+    }
+
+    async getRequestParameters(text, parameters, prompt) {
+        const requestParameters = await super.getRequestParameters(text, parameters, prompt);
+
+        let search_parameters = {};
+        if (parameters.search_parameters) {
+            try {
+                search_parameters = JSON.parse(parameters.search_parameters);
+            } catch (error) {
+                throw new Error(`Invalid 'search_parameters' parameter: ${error.message}`);
+            }
+        }
+
+        // Validate search parameters before including them
+        if (Object.keys(search_parameters).length > 0) {
+            this.validateSearchParameters(search_parameters);
+        }
+
+        // only set search_parameters if it's not undefined or empty
+        if (Object.keys(search_parameters).length > 0) {
+            requestParameters.search_parameters = search_parameters;
+        }
+
+        return requestParameters;
+    }
+
+    async execute(text, parameters, prompt, cortexRequest) {
+        const requestParameters = await this.getRequestParameters(text, parameters, prompt);
+        const { stream } = parameters;
+
+        cortexRequest.data = {
+            ...(cortexRequest.data || {}),
+            ...requestParameters,
+        };
+        cortexRequest.params = {}; // query params
+        cortexRequest.stream = stream;
+
+        return this.executeRequest(cortexRequest);
+
+    }
+
+    // Override processStreamEvent to handle Grok streaming format
+    processStreamEvent(event, requestProgress) {
+        // First, let the parent handle the basic streaming logic
+        const processedProgress = super.processStreamEvent(event, requestProgress);
+        
+        return processedProgress;
+    }
+
+    // Override tryParseMessages to preserve X.AI vision detail field
+    async tryParseMessages(messages) {
+        return await Promise.all(messages.map(async message => {
+            try {
+                // Handle tool-related message types
+                if (message.role === "tool" || (message.role === "assistant" && message.tool_calls)) {
+                    return {
+                        ...message
+                    };
+                }
+
+                if (Array.isArray(message.content)) {
+                    return {
+                        ...message,
+                        content: await Promise.all(message.content.map(async item => {
+                            const parsedItem = safeJsonParse(item);
+
+                            if (typeof parsedItem === 'string') {
+                                return { type: 'text', text: parsedItem };
+                            }
+
+                            if (typeof parsedItem === 'object' && parsedItem !== null) {
+                                // Handle both 'image' and 'image_url' types
+                                if (parsedItem.type === 'image' || parsedItem.type === 'image_url') {
+                                    const url = parsedItem.url || parsedItem.image_url?.url;
+                                    const detail = parsedItem.image_url?.detail || parsedItem.detail;
+                                    if (url && await this.validateImageUrl(url)) {
+                                        const imageUrl = { url };
+                                        if (detail) {
+                                            imageUrl.detail = detail;
+                                        }
+                                        return { type: 'image_url', image_url: imageUrl };
+                                    }
+                                    return { type: 'text', text: typeof item === 'string' ? item : JSON.stringify(item) };
+                                }
+                            }
+                            
+                            return parsedItem;
+                        }))
+                    };
+                }
+            } catch (e) {
+                return message;
+            }
+            return message;
+        }));
+    }
+
+    // Override parseResponse to handle Grok-specific response fields
+    parseResponse(data) {
+        if (!data) return "";
+        const { choices } = data;
+        if (!choices || !choices.length) {
+            return data;
+        }
+
+        // if we got a choices array back with more than one choice, return the whole array
+        if (choices.length > 1) {
+            return choices;
+        }
+
+        const choice = choices[0];
+        const message = choice.message;
+
+        // Create standardized CortexResponse object
+        const cortexResponse = new CortexResponse({
+            output_text: message.content || "",
+            finishReason: choice.finish_reason || 'stop',
+            usage: data.usage || null,
+            metadata: {
+                model: this.modelName
+            }
+        });
+
+        // Handle tool calls
+        if (message.tool_calls) {
+            cortexResponse.toolCalls = message.tool_calls;
+        }
+
+        // Handle Grok-specific Live Search data
+        if (data.citations) {
+            cortexResponse.citations = data.citations.map(url => ({
+                title: extractCitationTitle(url),
+                url: url,
+                content: extractCitationTitle(url)
+            }));
+        }
+
+        if (data.search_queries) {
+            cortexResponse.searchQueries = data.search_queries;
+        }
+
+        if (data.web_search_results) {
+            cortexResponse.searchResults = data.web_search_results;
+        }
+
+        if (data.real_time_data) {
+            cortexResponse.realTimeData = data.real_time_data;
+        }
+
+        // Return the CortexResponse object
+        return cortexResponse;
+    }
+
+}
+
+export default GrokVisionPlugin; 

package/server/plugins/modelPlugin.js

@@ -608,7 +608,6 @@ class ModelPlugin {
             let parsedMessage;
             try {
                 parsedMessage = JSON.parse(event.data);
-                requestProgress.data = event.data;
             } catch (error) {
                 throw new Error(`Could not parse stream data: ${error}`);
             }
@@ -619,6 +618,9 @@ class ModelPlugin {
                 throw new Error(streamError);
             }
 
+            // Set the data for the event
+            requestProgress.data = event.data;
+
             // finish reason can be in different places in the message
             const finishReason = parsedMessage?.choices?.[0]?.finish_reason || parsedMessage?.candidates?.[0]?.finishReason;
             if (finishReason) {

package/server/plugins/openAiChatPlugin.js

@@ -1,6 +1,7 @@
 // OpenAIChatPlugin.js
 import ModelPlugin from './modelPlugin.js';
 import logger from '../../lib/logger.js';
+import CortexResponse from '../../lib/cortexResponse.js';
 
 class OpenAIChatPlugin extends ModelPlugin {
     constructor(pathway, model) {
@@ -45,15 +46,24 @@ class OpenAIChatPlugin extends ModelPlugin {
     // Set up parameters specific to the OpenAI Chat API
     getRequestParameters(text, parameters, prompt) {
         const { modelPromptText, modelPromptMessages, tokenLength, modelPrompt } = this.getCompiledPrompt(text, parameters, prompt);
-        const { stream } = parameters;
-    
+        let { stream, tools, functions } = parameters;
+
+        if (typeof tools === 'string') {
+            tools = JSON.parse(tools);
+        }
+        
+        if (typeof functions === 'string') {
+            functions = JSON.parse(functions);
+        }
+
         // Define the model's max token length
         const modelTargetTokenLength = this.getModelMaxPromptTokens();
-    
+
         let requestMessages = modelPromptMessages || [{ "role": "user", "content": modelPromptText }];
         
         // Check if the messages are in Palm format and convert them to OpenAI format if necessary
         const isPalmFormat = requestMessages.some(message => 'author' in message);
+
         if (isPalmFormat) {
             const context = modelPrompt.context || '';
             const examples = modelPrompt.examples || [];
@@ -65,11 +75,13 @@ class OpenAIChatPlugin extends ModelPlugin {
             // Remove older messages until the token length is within the model's limit
             requestMessages = this.truncateMessagesToTargetLength(requestMessages, modelTargetTokenLength);
         }
-    
+
         const requestParameters = {
         messages: requestMessages,
         temperature: this.temperature ?? 0.7,
         ...(stream !== undefined ? { stream } : {}),
+        ...(tools && tools.length > 0 ? { tools, tool_choice: parameters.tool_choice || 'auto' } : {}),
+        ...(functions && functions.length > 0 ? { functions } : {}),
         };
     
         return requestParameters;
@@ -80,7 +92,7 @@ class OpenAIChatPlugin extends ModelPlugin {
         const requestParameters = this.getRequestParameters(text, parameters, prompt);
 
         cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
-        cortexRequest.params = {}; // query params
+        cortexRequest.params = {};
 
         return this.executeRequest(cortexRequest);
     }
@@ -98,9 +110,86 @@ class OpenAIChatPlugin extends ModelPlugin {
             return choices;
         }
 
-        // otherwise, return the first choice
-        const messageResult = choices[0].message && choices[0].message.content && choices[0].message.content.trim();
-        return messageResult ?? null;
+        const choice = choices[0];
+        const message = choice.message;
+        if (!message) {
+            return null;
+        }
+
+        // Create standardized CortexResponse object
+        const cortexResponse = new CortexResponse({
+            output_text: message.content || "",
+            finishReason: choice.finish_reason || 'stop',
+            usage: data.usage || null,
+            metadata: {
+                model: this.modelName
+            }
+        });
+
+        // Handle tool calls
+        if (message.tool_calls) {
+            cortexResponse.toolCalls = message.tool_calls;
+        } else if (message.function_call) {
+            cortexResponse.functionCall = message.function_call;
+        }
+
+        return cortexResponse;
+    }
+
+    // Override processStreamEvent to handle OpenAI Chat streaming format
+    processStreamEvent(event, requestProgress) {
+        // check for end of stream or in-stream errors
+        if (event.data.trim() === '[DONE]') {
+            requestProgress.progress = 1;
+        } else {
+            let parsedMessage;
+            try {
+                parsedMessage = JSON.parse(event.data);
+            } catch (error) {
+                throw new Error(`Could not parse stream data: ${error}`);
+            }
+
+            // error can be in different places in the message
+            const streamError = parsedMessage?.error || parsedMessage?.choices?.[0]?.delta?.content?.error || parsedMessage?.choices?.[0]?.text?.error;
+            if (streamError) {
+                throw new Error(streamError);
+            }
+
+            // Check if this is an empty/idle event that we should skip
+            const delta = parsedMessage?.choices?.[0]?.delta;
+            const isEmptyEvent = !delta || 
+                (Object.keys(delta).length === 0) || 
+                (Object.keys(delta).length === 1 && delta.content === '') ||
+                (Object.keys(delta).length === 1 && delta.tool_calls && delta.tool_calls.length === 0);
+            
+            // Skip publishing empty events unless they have a finish_reason
+            const hasFinishReason = parsedMessage?.choices?.[0]?.finish_reason || parsedMessage?.candidates?.[0]?.finishReason;
+            
+            if (isEmptyEvent && !hasFinishReason) {
+                // Return requestProgress without setting data to prevent publishing
+                return requestProgress;
+            }
+            
+            // Set the data for non-empty events or events with finish_reason
+            requestProgress.data = event.data;
+
+            // finish reason can be in different places in the message
+            const finishReason = parsedMessage?.choices?.[0]?.finish_reason || parsedMessage?.candidates?.[0]?.finishReason;
+            if (finishReason) {
+                switch (finishReason.toLowerCase()) {
+                    case 'safety':
+                        const safetyRatings = JSON.stringify(parsedMessage?.candidates?.[0]?.safetyRatings) || '';
+                        logger.warn(`Request ${this.requestId} was blocked by the safety filter. ${safetyRatings}`);
+                        requestProgress.data = `\n\nResponse blocked by safety filter: ${safetyRatings}`;
+                        requestProgress.progress = 1;
+                        break;
+                    default:
+                        requestProgress.progress = 1;
+                        break;
+                }
+            }
+        }
+        return requestProgress;
     }
 
     // Override the logging function to display the messages and responses
@@ -135,11 +224,15 @@ class OpenAIChatPlugin extends ModelPlugin {
     
         if (stream) {
             logger.info(`[response received as an SSE stream]`);
-        } else {
-            const responseText = this.parseResponse(responseData);
-            const { length, units } = this.getLength(responseText);
-            logger.info(`[response received containing ${length} ${units}]`);
-            logger.verbose(`${this.shortenContent(responseText)}`);
+        } else {           
+            if (typeof responseData === 'string') {
+                const { length, units } = this.getLength(responseData);
+                logger.info(`[response received containing ${length} ${units}]`);
+                logger.verbose(`${this.shortenContent(responseData)}`);
+            } else {
+                logger.info(`[response received containing object]`);
+                logger.verbose(`${JSON.stringify(responseData)}`);
+            }
         }
 
         prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);