@aj-archipelago/cortex 1.3.62 → 1.3.63

package/pathways/system/rest_streaming/{sys_claude_35_sonnet.js → sys_openai_chat_gpt4_omni.js}

@@ -1,5 +1,5 @@
-// sys_claude_35_sonnet.js
-// override handler for claude-35-sonnet
+// sys_openai_chat_gpt4_omni.js
+// override handler for gpt-4-omni
 
 import { Prompt } from '../../../server/prompt.js';
 
@@ -12,8 +12,10 @@ export default {
     ],
     inputParameters: {
         messages: [{role: '', content: []}],
+        tools: '',
+        tool_choice: 'auto',
     },
-    model: 'claude-35-sonnet-vertex',
+    model: 'oai-gpt4o',
     useInputChunking: false,
-    emulateOpenAIChatModel: 'claude-3.5-sonnet',
+    emulateOpenAIChatModel: 'gpt-4o',
 }

package/pathways/system/rest_streaming/sys_openai_chat_gpt4_omni_mini.js

@@ -0,0 +1,21 @@
+// sys_openai_chat_gpt4_omni_mini.js
+// override handler for gpt-4-omni-mini
+
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+    [
+        new Prompt({ messages: [
+            "{{messages}}",
+        ]}),
+    ],
+    inputParameters: {
+        messages: [{role: '', content: []}],
+        tools: '',
+        tool_choice: 'auto',
+    },
+    model: 'oai-gpt4o-mini',
+    useInputChunking: false,
+    emulateOpenAIChatModel: 'gpt-4o-mini',
+}

package/pathways/system/rest_streaming/{sys_claude_3_haiku.js → sys_openai_chat_gpt5.js}

@@ -1,5 +1,5 @@
-// sys_claude_3_haiku.js
-// override handler for claude-3-haiku
+// sys_openai_chat_gpt5.js
+// override handler for gpt-5
 
 import { Prompt } from '../../../server/prompt.js';
 
@@ -12,8 +12,10 @@ export default {
     ],
     inputParameters: {
         messages: [{role: '', content: []}],
+        tools: '',
+        tool_choice: 'auto',
     },
-    model: 'claude-3-haiku-vertex',
+    model: 'oai-gpt5',
     useInputChunking: false,
-    emulateOpenAIChatModel: 'claude-3-haiku',
-}
+    emulateOpenAIChatModel: 'gpt-5',
+} 

package/pathways/system/rest_streaming/sys_openai_chat_gpt5_chat.js

@@ -0,0 +1,21 @@
+// sys_openai_chat_gpt5_chat.js
+// override handler for gpt-5-chat
+
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+    [
+        new Prompt({ messages: [
+            "{{messages}}",
+        ]}),
+    ],
+    inputParameters: {
+        messages: [{role: '', content: []}],
+        tools: '',
+        tool_choice: 'auto',
+    },
+    model: 'oai-gpt5-chat',
+    useInputChunking: false,
+    emulateOpenAIChatModel: 'gpt-5-chat',
+} 

package/pathways/system/rest_streaming/sys_openai_chat_gpt5_mini.js

@@ -0,0 +1,21 @@
+// sys_openai_chat_gpt5_mini.js
+// override handler for gpt-5-mini
+
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+    [
+        new Prompt({ messages: [
+            "{{messages}}",
+        ]}),
+    ],
+    inputParameters: {
+        messages: [{role: '', content: []}],
+        tools: '',
+        tool_choice: 'auto',
+    },
+    model: 'oai-gpt5-mini',
+    useInputChunking: false,
+    emulateOpenAIChatModel: 'gpt-5-mini',
+} 

package/pathways/system/rest_streaming/sys_openai_chat_gpt5_nano.js

@@ -0,0 +1,21 @@
+// sys_openai_chat_gpt5_nano.js
+// override handler for gpt-5-nano
+
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+    [
+        new Prompt({ messages: [
+            "{{messages}}",
+        ]}),
+    ],
+    inputParameters: {
+        messages: [{role: '', content: []}],
+        tools: '',
+        tool_choice: 'auto',
+    },
+    model: 'oai-gpt5-nano',
+    useInputChunking: false,
+    emulateOpenAIChatModel: 'gpt-5-nano',
+} 

package/pathways/system/rest_streaming/{sys_openai_chat_o1.js → sys_openai_chat_o3.js}

@@ -1,4 +1,4 @@
-// sys_openai_chat_o1.js
+// sys_openai_chat_o3.js
 
 import { Prompt } from '../../../server/prompt.js';
 
@@ -11,9 +11,12 @@ export default {
     ],
     inputParameters: {
         messages: [{role: '', content: []}],
+        functions: '',
+        tools: '',
+        tool_choice: 'auto',
     },
-    model: 'oai-o1',
+    model: 'oai-o3',
     useInputChunking: false,
-    emulateOpenAIChatModel: 'o1',
+    emulateOpenAIChatModel: 'o3',
     enableDuplicateRequests: false,
 }

package/pathways/system/rest_streaming/sys_openai_chat_o3_mini.js

@@ -11,6 +11,9 @@ export default {
     ],
     inputParameters: {
         messages: [{role: '', content: []}],
+        functions: '',
+        tools: '',
+        tool_choice: 'auto',
     },
     model: 'oai-o3-mini',
     useInputChunking: false,

package/pathways/system/workspaces/run_workspace_prompt.js

@@ -0,0 +1,99 @@
+import { config } from '../../../config.js';
+import { chatArgsHasImageUrl, chatArgsHasType, getAvailableFiles, removeOldImageAndFileContent } from '../../../lib/util.js';
+import { loadEntityConfig } from '../../../pathways/system/entity/tools/shared/sys_entity_tools.js';
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    emulateOpenAIChatModel: 'cortex-agent',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    useSingleTokenStream: false,
+
+    inputParameters: {
+        prompt: "",
+        systemPrompt: "",
+        chatHistory: [{role: '', content: []}],
+        text: "",
+        entityId: "labeeb",
+        aiName: "Jarvis",
+        language: "English",
+        model: "oai-gpt41", // Allow user to specify model
+    },
+
+    timeout: 600,
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        let pathwayResolver = resolver;
+
+        // Load input parameters and information into args
+        const { entityId, aiName, language, model } = { ...pathwayResolver.pathway.inputParameters, ...args };
+        
+        const entityConfig = loadEntityConfig(entityId);
+
+        // Initialize chat history if needed
+        if (!args.chatHistory || args.chatHistory.length === 0) {
+            args.chatHistory = [];
+        }
+
+        // Limit the chat history to 20 messages to speed up processing
+        args.chatHistory = args.chatHistory.slice(-20);
+
+        // Add entity constants for template rendering
+        args = {
+            ...args,
+            ...config.get('entityConstants'),
+            entityId,
+            aiName,
+            language,
+            model
+        };
+
+        // Extract available files from chat history
+        const availableFiles = getAvailableFiles(args.chatHistory);
+
+        // Check for both image and file content (CSV files have type 'file', not 'image_url')
+        const hasImageContent = chatArgsHasImageUrl(args);
+        const hasFileContent = chatArgsHasType(args, 'file');
+        const visionContentPresent = hasImageContent || hasFileContent;
+
+        // Remove old image and file content while preserving the latest uploads
+        visionContentPresent && (args.chatHistory = removeOldImageAndFileContent(args.chatHistory));
+
+        const promptMessages = [
+            {"role": "system", "content": `${args.systemPrompt || "Assistant is an expert journalist's assistant for Al Jazeera Media Network. When a user posts a request, Assistant will come up with the best response while upholding the highest journalistic standards."}\n\n{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_AVAILABLE_FILES}}\n\n{{renderTemplate AI_DATETIME}}`},
+            "{{chatHistory}}"
+        ];
+
+        // Only add a user message if there's actual text or prompt content
+        const userContent = `${args.text || ""}\n\n${args.prompt || ""}`.trim();
+        if (userContent) {
+            promptMessages.push({"role": "user", "content": userContent});
+        }
+
+        pathwayResolver.pathwayPrompt = [
+            new Prompt({ messages: promptMessages }),
+        ];
+
+        pathwayResolver.args = {...args};
+
+        try {
+            let currentMessages = JSON.parse(JSON.stringify(args.chatHistory));
+
+            console.log("currentMessages", currentMessages);
+            console.log("args", args);
+
+            let response = await runAllPrompts({
+                ...args,
+                chatHistory: currentMessages,
+                availableFiles,
+                model: args.model // Pass the model from args
+            });
+
+            return response;
+
+        } catch (e) {
+            pathwayResolver.logError(e);
+            throw e;
+        }
+    }
+};

package/pathways/vision.js

@@ -11,9 +11,9 @@ export default {
     inputParameters: {
         chatHistory: [{role: '', content: []}],
         contextId: ``,
+        model: `oai-gpt41`,
     },
     max_tokens: 1024,
-    model: 'oai-gpt41',
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,

package/server/graphql.js

@@ -18,7 +18,7 @@ import cors from 'cors';
 import { buildModels, buildPathways } from '../config.js';
 import logger from '../lib/logger.js';
 import { buildModelEndpoints } from '../lib/requestExecutor.js';
-import { startTestServer } from '../tests/server.js';
+import { startTestServer } from '../tests/helpers/server.js';
 import { requestState } from './requestState.js';
 import { cancelRequestResolver } from './resolver.js';
 import subscriptions from './subscriptions.js';

package/server/modelExecutor.js

@@ -33,7 +33,9 @@ import ApptekTranslatePlugin from './plugins/apptekTranslatePlugin.js';
 import GoogleTranslatePlugin from './plugins/googleTranslatePlugin.js';
 import GroqChatPlugin from './plugins/groqChatPlugin.js';
 import VeoVideoPlugin from './plugins/veoVideoPlugin.js';
+import GrokVisionPlugin from './plugins/grokVisionPlugin.js';
 import AzureFoundryAgentsPlugin from './plugins/azureFoundryAgentsPlugin.js';
+import GoogleCsePlugin from './plugins/googleCsePlugin.js';
 
 class ModelExecutor {
     constructor(pathway, model) {
@@ -134,9 +136,15 @@ class ModelExecutor {
             case 'VEO-VIDEO':
                 plugin = new VeoVideoPlugin(pathway, model);
                 break;
+            case 'GROK-VISION':
+                plugin = new GrokVisionPlugin(pathway, model);
+                break;
             case 'AZURE-FOUNDRY-AGENTS':
                 plugin = new AzureFoundryAgentsPlugin(pathway, model);
                 break;
+            case 'GOOGLE-CSE':
+                plugin = new GoogleCsePlugin(pathway, model);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }

package/server/pathwayResolver.js

@@ -29,7 +29,7 @@ class PathwayResolver {
         this.requestId = uuidv4();
         this.rootRequestId = null;
         this.responseParser = new PathwayResponseParser(pathway);
-        this.tool = null;
+        this.pathwayResultData = {};
         this.modelName = [
             pathway.model,
             args?.model,
@@ -72,12 +72,58 @@ class PathwayResolver {
         // set up initial prompt
         this.pathwayPrompt = pathway.prompt;
     }
+    
+    // Legacy 'tool' property is now stored in pathwayResultData
+    get tool() {      
+        // Select fields to serialize for legacy compat, excluding undefined values
+        const legacyFields = Object.fromEntries(
+            Object.entries({
+                hideFromModel: this.pathwayResultData.hideFromModel,    
+                toolCallbackName: this.pathwayResultData.toolCallbackName, 
+                title: this.pathwayResultData.title,
+                search: this.pathwayResultData.search,
+                coding: this.pathwayResultData.coding,
+                codeRequestId: this.pathwayResultData.codeRequestId,
+                toolCallbackId: this.pathwayResultData.toolCallbackId,
+                toolUsed: this.pathwayResultData.toolUsed,
+                citations: this.pathwayResultData.citations,
+
+            }).filter(([_, value]) => value !== undefined)
+        );
+        return JSON.stringify(legacyFields);
+    }
+
+    set tool(value) {
+        // Accepts a JSON string, parses, merges into pathwayResultData
+        let parsed;
+        try {
+            parsed = (typeof value === 'string') ? JSON.parse(value) : value;
+            this.pathwayResultData = this.mergeResultData(parsed);
+        } catch (e) {
+            // Optionally warn: invalid format or merge error
+            console.warn('Invalid tool property assignment:', e);
+        }
+    }
 
     publishNestedRequestProgress(requestProgress) {
-        if (requestProgress.progress === 1 && this.rootRequestId) {
-            delete requestProgress.progress;
+
+        if (this.rootRequestId) {
+            // if this is a nested request, don't end the stream
+            if (requestProgress.progress === 1) {
+                delete requestProgress.progress;
+            }
+            publishRequestProgress(requestProgress);
+        } else {
+            // this is a root request, so we add the pathwayResultData to the info
+            // and allow the end stream message to be sent
+            if (requestProgress.progress === 1) {
+                const infoObject = { ...this.pathwayResultData || {} };
+                requestProgress.info = JSON.stringify(infoObject);
+                requestProgress.error = this.errors.join(', ') || '';
+            }
+            publishRequestProgress(requestProgress);
         }
-        publishRequestProgress({...requestProgress, info: this.tool || '', error: this.errors.join(', ')});
+
     }
 
     // This code handles async and streaming responses for either long-running
@@ -120,31 +166,102 @@ class PathwayResolver {
             
             // some models don't support progress updates
             if (!modelTypesExcludedFromProgressUpdates.includes(this.model.type)) {
+                const infoObject = { ...this.pathwayResultData || {} };
                 this.publishNestedRequestProgress({
                         requestId: this.rootRequestId || this.requestId,
                         progress: Math.min(completedCount, totalCount) / totalCount,
                         // Clients expect these to be strings
                         data: JSON.stringify(responseData || ''),
-                        info: this.tool || '',
+                        info: JSON.stringify(infoObject) || '',
                         error: this.errors.join(', ') || ''
                 });
             }
         }
     }
 
-    mergeResults(mergeData) {
-        if (mergeData) {
-            this.previousResult = mergeData.previousResult ? mergeData.previousResult : this.previousResult;
-            this.warnings = [...this.warnings, ...(mergeData.warnings || [])];
-            this.errors = [...this.errors, ...(mergeData.errors || [])];
-            try {
-                const mergeDataTool = typeof mergeData.tool === 'string' ? JSON.parse(mergeData.tool) : mergeData.tool || {};
-                const thisTool = typeof this.tool === 'string' ? JSON.parse(this.tool) : this.tool || {};
-                this.tool = JSON.stringify({ ...thisTool, ...mergeDataTool });
-            } catch (error) {
-                logger.warn('Error merging pathway resolver tool objects: ' + error);
+    mergeResolver(otherResolver) {
+        if (otherResolver) {
+            this.previousResult = otherResolver.previousResult ? otherResolver.previousResult : this.previousResult;
+            this.warnings = [...this.warnings, ...otherResolver.warnings];
+            this.errors = [...this.errors, ...otherResolver.errors];
+
+            // Use the shared mergeResultData method
+            this.pathwayResultData = this.mergeResultData(otherResolver.pathwayResultData);
+        }
+    }
+
+    // Merge pathwayResultData with either another pathwayResultData object or a CortexResponse
+    mergeResultData(newData) {
+        if (!newData) return this.pathwayResultData;
+
+        const currentData = this.pathwayResultData || {};
+
+        // Handle CortexResponse objects
+        if (newData.constructor && newData.constructor.name === 'CortexResponse') {
+            const cortexResponse = newData;
+            const cortexData = {
+                citations: cortexResponse.citations,
+                toolCalls: cortexResponse.toolCalls,
+                functionCall: cortexResponse.functionCall,
+                usage: cortexResponse.usage,
+                finishReason: cortexResponse.finishReason
+            };
+            newData = cortexData;
+        }
+
+        // Create merged result
+        const merged = { ...currentData, ...newData };
+
+        // Handle array fields that should be concatenated
+        const arrayFields = ['citations', 'toolCalls'];
+        for (const field of arrayFields) {
+            const currentArray = currentData[field] || [];
+            const newArray = newData[field] || [];
+            if (newArray.length > 0) {
+                merged[field] = [...currentArray, ...newArray];
+            } else if (currentArray.length > 0) {
+                merged[field] = currentArray;
             }
         }
+
+        // Handle usage and toolUsed data - convert to arrays with most recent first
+        const createArrayFromData = (currentValue, newValue) => {
+            if (!currentValue && !newValue) return null;
+
+            const array = [];
+
+            // Add new value first (most recent)
+            if (newValue) {
+                if (Array.isArray(newValue)) {
+                    array.push(...newValue);
+                } else {
+                    array.push(newValue);
+                }
+            }
+
+            // Add current value second (older)
+            if (currentValue) {
+                if (Array.isArray(currentValue)) {
+                    array.push(...currentValue);
+                } else {
+                    array.push(currentValue);
+                }
+            }
+
+            return array;
+        };
+
+        const usageArray = createArrayFromData(currentData.usage, newData.usage);
+        if (usageArray) {
+            merged.usage = usageArray;
+        }
+
+        const toolUsedArray = createArrayFromData(currentData.toolUsed, newData.toolUsed);
+        if (toolUsedArray) {
+            merged.toolUsed = toolUsedArray;
+        }
+
+        return merged;
     }
 
     async handleStream(response) {
@@ -251,6 +368,15 @@ class PathwayResolver {
     }
 
     async promptAndParse(args) {
+        // Check if model is specified in args and swap if different from current model
+        if (args.modelOverride && args.modelOverride !== this.modelName) {
+            try {
+                this.swapModel(args.modelOverride);
+            } catch (error) {
+                this.logError(`Failed to swap model to ${args.modelOverride}: ${error.message}`);
+            }
+        }
+
         // Get saved context from contextId or change contextId if needed
         const { contextId } = args;
         this.savedContextId = contextId ? contextId : uuidv4();
@@ -500,6 +626,30 @@ class PathwayResolver {
         return result;
     }
 
+    /**
+     * Swaps the model used by this PathwayResolver
+     * @param {string} newModelName - The name of the new model to use
+     * @throws {Error} If the new model is not found in the endpoints
+     */
+    swapModel(newModelName) {
+        // Validate that the new model exists in endpoints
+        if (!this.endpoints[newModelName]) {
+            throw new Error(`Model ${newModelName} not found in config`);
+        }
+
+        // Update model references
+        this.modelName = newModelName;
+        this.model = this.endpoints[newModelName];
+
+        // Create new ModelExecutor with the new model
+        this.modelExecutor = new ModelExecutor(this.pathway, this.model);
+
+        // Recalculate chunk max token length as it depends on the model
+        this.chunkMaxTokenLength = this.getChunkMaxTokenLength();
+
+        this.logWarning(`Model swapped to ${newModelName}`);
+    }
+
     async applyPrompt(prompt, text, parameters) {
         if (requestState[this.requestId].canceled) {
             return;

package/server/pathwayResponseParser.js

@@ -1,4 +1,5 @@
 import { parseNumberedList, parseNumberedObjectList, parseCommaSeparatedList, isCommaSeparatedList, isNumberedList, parseJson } from './parser.js';
+import CortexResponse from '../lib/cortexResponse.js';
 
 class PathwayResponseParser {
     constructor(pathway) {
@@ -6,24 +7,31 @@ class PathwayResponseParser {
     }
 
     async parse(data) {
+        
+        let dataToParse = data;
+        
+        if (data instanceof CortexResponse) {
+            dataToParse = data.toString();
+        }       
+
         if (this.pathway.parser) {
-            return this.pathway.parser(data);
+            return this.pathway.parser(dataToParse);
         }
 
         if (this.pathway.list) {
-            if (isNumberedList(data)) {
+            if (isNumberedList(dataToParse)) {
                 if (this.pathway.format) {
-                    return await parseNumberedObjectList(data, this.pathway.format);
+                    return await parseNumberedObjectList(dataToParse, this.pathway.format);
                 }
-                return parseNumberedList(data);
-            } else if (isCommaSeparatedList(data)) {
-                return parseCommaSeparatedList(data);
+                return parseNumberedList(dataToParse);
+            } else if (isCommaSeparatedList(dataToParse)) {
+                return parseCommaSeparatedList(dataToParse);
             }
-            return [data];
+            return [dataToParse];
         }
 
         if (this.pathway.json) {
-            return await parseJson(data);
+            return await parseJson(dataToParse);
         }
 
         return data;

package/server/plugins/azureFoundryAgentsPlugin.js

@@ -86,7 +86,7 @@ class AzureFoundryAgentsPlugin extends ModelPlugin {
             ...(parameters.top_p && { top_p: parameters.top_p }),
             ...(parameters.tool_choice && { tool_choice: parameters.tool_choice }),
             ...(parameters.response_format && { response_format: parameters.response_format }),
-            ...(parameters.parallel_tool_calls && { parallel_tool_calls: parameters.parallel_tool_calls }),
+            ...(parameters.parallel_tool_calls !== undefined && { parallel_tool_calls: parameters.parallel_tool_calls }),
             ...(parameters.truncation_strategy && { truncation_strategy: parameters.truncation_strategy })
         };