@aj-archipelago/cortex 1.3.35 → 1.3.36

package/server/plugins/openAiReasoningVisionPlugin.js

@@ -0,0 +1,48 @@
+import OpenAIVisionPlugin from './openAiVisionPlugin.js';
+
+class OpenAIReasoningVisionPlugin extends OpenAIVisionPlugin {
+    
+    async tryParseMessages(messages) {
+        const parsedMessages = await super.tryParseMessages(messages);
+
+        let newMessages = [];
+
+        newMessages = parsedMessages.map(message => ({
+            role: message.role === 'system' ? 'developer' : message.role,
+            content: message.content
+        })).filter(message => ['user', 'assistant', 'developer', 'tool'].includes(message.role));
+
+        return newMessages;
+    }
+
+    async getRequestParameters(text, parameters, prompt) {
+        const requestParameters = await super.getRequestParameters(text, parameters, prompt);
+
+        const modelMaxReturnTokens = this.getModelMaxReturnTokens();
+        const maxTokensPrompt = this.promptParameters.max_tokens;
+        const maxTokensModel = this.getModelMaxTokenLength() * (1 - this.getPromptTokenRatio());
+
+        const maxTokens = maxTokensPrompt || maxTokensModel;
+
+        delete requestParameters.max_tokens;
+        requestParameters.max_completion_tokens = maxTokens ? Math.min(maxTokens, modelMaxReturnTokens) : modelMaxReturnTokens;
+        requestParameters.temperature = 1;
+
+        if (this.promptParameters.reasoningEffort) {
+            const effort = this.promptParameters.reasoningEffort.toLowerCase();
+            if (['high', 'medium', 'low'].includes(effort)) {
+                requestParameters.reasoning_effort = effort;
+            } else {
+                requestParameters.reasoning_effort = 'low';
+            }
+        }
+        
+        if (this.promptParameters.responseFormat) {
+            requestParameters.response_format = this.promptParameters.responseFormat;
+        }
+
+        return requestParameters;
+    }
+}
+
+export default OpenAIReasoningVisionPlugin; 

package/server/plugins/openAiVisionPlugin.js

@@ -1,5 +1,6 @@
 import OpenAIChatPlugin from './openAiChatPlugin.js';
 import logger from '../../lib/logger.js';
+import { requestState } from '../requestState.js';
 
 function safeJsonParse(content) {
     try {
@@ -15,14 +16,31 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
     constructor(pathway, model) {
         super(pathway, model);
         this.isMultiModal = true;
+        this.pathwayToolCallback = pathway.toolCallback;
+        this.toolCallsBuffer = [];
+        this.contentBuffer = ''; // Initialize content buffer
     }
     
     async tryParseMessages(messages) {
         return await Promise.all(messages.map(async message => {
             try {
+                // Handle tool-related message types
                 if (message.role === "tool") {
-                    return message;
+                    return {
+                        role: message.role,
+                        content: message.content,
+                        tool_call_id: message.tool_call_id
+                    };
                 }
+
+                if (message.role === "assistant" && message.tool_calls) {
+                    return {
+                        role: message.role,
+                        content: message.content,
+                        tool_calls: message.tool_calls
+                    };
+                }
+
                 if (Array.isArray(message.content)) {
                     message.content = await Promise.all(message.content.map(async item => {
                         const parsedItem = safeJsonParse(item);
@@ -36,7 +54,7 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
                             if (url && await this.validateImageUrl(url)) {
                                 return {type: parsedItem.type, image_url: {url}};
                             }
-                            return { type: 'text', text: 'Image skipped: unsupported format' };
+                            return { type: 'text', text: typeof item === 'string' ? item : JSON.stringify(item) };
                         }
                         
                         return parsedItem;
@@ -90,14 +108,19 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
             logger.info(`[request sent containing ${length} ${units}]`);
             logger.verbose(`${this.shortenContent(content)}`);
         }
-    
         if (stream) {
             logger.info(`[response received as an SSE stream]`);
         } else {
-            const responseText = this.parseResponse(responseData);
-            const { length, units } = this.getLength(responseText);
-            logger.info(`[response received containing ${length} ${units}]`);
-            logger.verbose(`${this.shortenContent(responseText)}`);
+            const parsedResponse = this.parseResponse(responseData);
+            
+            if (typeof parsedResponse === 'string') {
+                const { length, units } = this.getLength(parsedResponse);
+                logger.info(`[response received containing ${length} ${units}]`);
+                logger.verbose(`${this.shortenContent(parsedResponse)}`);
+            } else {
+                logger.info(`[response received containing object]`);
+                logger.verbose(`${JSON.stringify(parsedResponse)}`);
+            }
         }
 
         prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
@@ -109,6 +132,15 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
 
         requestParameters.messages = await this.tryParseMessages(requestParameters.messages);
 
+        // Add tools support if provided in parameters
+        if (parameters.tools) {
+            requestParameters.tools = parameters.tools;
+        }
+
+        if (parameters.tool_choice) {
+            requestParameters.tool_choice = parameters.tool_choice;
+        }
+
         const modelMaxReturnTokens = this.getModelMaxReturnTokens();
         const maxTokensPrompt = this.promptParameters.max_tokens;
         const maxTokensModel = this.getModelMaxTokenLength() * (1 - this.getPromptTokenRatio());
@@ -138,6 +170,159 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
         return this.executeRequest(cortexRequest);
     }
 
+    // Override parseResponse to handle tool calls
+    parseResponse(data) {
+        if (!data) return "";
+        const { choices } = data;
+        if (!choices || !choices.length) {
+            return data;
+        }
+
+        // if we got a choices array back with more than one choice, return the whole array
+        if (choices.length > 1) {
+            return choices;
+        }
+
+        const choice = choices[0];
+        const message = choice.message;
+
+        // Handle tool calls in the response
+        if (message.tool_calls) {
+            return {
+                role: message.role,
+                content: message.content || "",
+                tool_calls: message.tool_calls
+            };
+        }
+
+        return message.content || "";
+    }
+
+    processStreamEvent(event, requestProgress) {
+        // check for end of stream or in-stream errors
+        if (event.data.trim() === '[DONE]') {
+            requestProgress.progress = 1;
+            // Clear buffers when stream is done
+            this.toolCallsBuffer = [];
+            this.contentBuffer = ''; // Clear content buffer
+        } else {
+            let parsedMessage;
+            try {
+                parsedMessage = JSON.parse(event.data);
+                requestProgress.data = event.data;
+            } catch (error) {
+                // Clear buffers on error
+                this.toolCallsBuffer = [];
+                this.contentBuffer = '';
+                throw new Error(`Could not parse stream data: ${error}`);
+            }
+
+            // error can be in different places in the message
+            const streamError = parsedMessage?.error || parsedMessage?.choices?.[0]?.delta?.content?.error || parsedMessage?.choices?.[0]?.text?.error;
+            if (streamError) {
+                // Clear buffers on error
+                this.toolCallsBuffer = [];
+                this.contentBuffer = '';
+                throw new Error(streamError);
+            }
+
+            const delta = parsedMessage?.choices?.[0]?.delta;
+
+            // Accumulate content
+            if (delta?.content) {
+                this.contentBuffer += delta.content;
+            }
+
+            // Handle tool calls in streaming response
+            if (delta?.tool_calls) {
+                // Accumulate tool call deltas into the buffer
+                delta.tool_calls.forEach((toolCall) => {
+                    const index = toolCall.index;
+                    if (!this.toolCallsBuffer[index]) {
+                        this.toolCallsBuffer[index] = {
+                            id: toolCall.id || '',
+                            type: toolCall.type || 'function',
+                            function: {
+                                name: toolCall.function?.name || '',
+                                arguments: toolCall.function?.arguments || ''
+                            }
+                        };
+                    } else {
+                        if (toolCall.function?.name) {
+                            this.toolCallsBuffer[index].function.name += toolCall.function.name;
+                        }
+                        if (toolCall.function?.arguments) {
+                            this.toolCallsBuffer[index].function.arguments += toolCall.function.arguments;
+                        }
+                    }
+                });
+            }
+
+            // finish reason can be in different places in the message
+            const finishReason = parsedMessage?.choices?.[0]?.finish_reason || parsedMessage?.candidates?.[0]?.finishReason;
+            if (finishReason) {
+                const pathwayResolver = requestState[this.requestId]?.pathwayResolver; // Get resolver
+
+                switch (finishReason.toLowerCase()) {
+                    case 'tool_calls':
+                        // Process complete tool calls when we get the finish reason
+                        if (this.pathwayToolCallback && this.toolCallsBuffer.length > 0) {
+                            const toolMessage = {
+                                role: 'assistant',
+                                content: delta?.content || '', 
+                                tool_calls: this.toolCallsBuffer,
+                            };
+                            this.pathwayToolCallback(pathwayResolver?.args, toolMessage, pathwayResolver);
+                        }
+                        // Don't set progress to 1 for tool calls to keep stream open
+                        // Clear tool buffer after processing, but keep content buffer
+                        this.toolCallsBuffer = []; 
+                        break;
+                    case 'safety':
+                        const safetyRatings = JSON.stringify(parsedMessage?.candidates?.[0]?.safetyRatings) || '';
+                        logger.warn(`Request ${this.requestId} was blocked by the safety filter. ${safetyRatings}`);
+                        requestProgress.data = `\n\nResponse blocked by safety filter: ${safetyRatings}`;
+                        requestProgress.progress = 1;
+                        // Clear buffers on finish
+                        this.toolCallsBuffer = [];
+                        this.contentBuffer = '';
+                        break;
+                    default: // Includes 'stop' and other normal finish reasons
+                        // Look to see if we need to add citations to the response
+                        if (pathwayResolver && this.contentBuffer) {
+                            const regex = /:cd_source\[(.*?)\]/g;
+                            let match;
+                            const foundIds = [];
+                            while ((match = regex.exec(this.contentBuffer)) !== null) {
+                                // Ensure the capture group exists and is not empty
+                                if (match[1] && match[1].trim()) { 
+                                    foundIds.push(match[1].trim());
+                                }
+                            }
+
+                            if (foundIds.length > 0) {
+                                const {searchResults, tool} = pathwayResolver;
+                                logger.info(`Found referenced searchResultIds: ${foundIds.join(', ')}`);
+
+                                if (searchResults) {
+                                    const toolObj = typeof tool === 'string' ? JSON.parse(tool) : (tool || {});
+                                    toolObj.citations = searchResults
+                                        .filter(result => foundIds.includes(result.searchResultId));
+                                    pathwayResolver.tool = JSON.stringify(toolObj);
+                                }
+                            }
+                        }
+                        requestProgress.progress = 1;
+                        // Clear buffers on finish
+                        this.toolCallsBuffer = [];
+                        this.contentBuffer = '';
+                        break;
+                }
+            }
+        }
+        return requestProgress;
+    }
+
 }
 
 export default OpenAIVisionPlugin;

package/tests/agentic.test.js

@@ -0,0 +1,256 @@
+// agentic.test.js
+// Tests for the agentic entity system
+
+import test from 'ava';
+import serverFactory from '../index.js';
+import { createClient } from 'graphql-ws';
+import ws from 'ws';
+
+let testServer;
+let wsClient;
+
+test.before(async () => {
+  process.env.CORTEX_ENABLE_REST = 'true';
+  const { server, startServer } = await serverFactory();
+  startServer && await startServer();
+  testServer = server;
+
+  // Create WebSocket client for subscriptions
+  wsClient = createClient({
+    url: 'ws://localhost:4000/graphql',
+    webSocketImpl: ws,
+    retryAttempts: 3,
+    connectionParams: {},
+    on: {
+      error: (error) => {
+        console.error('WS connection error:', error);
+      }
+    }
+  });
+
+  // Test the connection by making a simple subscription
+  try {
+    await new Promise((resolve, reject) => {
+      const subscription = wsClient.subscribe(
+        {
+          query: `
+            subscription TestConnection {
+              requestProgress(requestIds: ["test"]) {
+                requestId
+              }
+            }
+          `
+        },
+        {
+          next: () => {
+            resolve();
+          },
+          error: reject,
+          complete: () => {
+            resolve();
+          }
+        }
+      );
+
+      // Add a timeout to avoid hanging
+      setTimeout(() => {
+        resolve();
+      }, 2000);
+    });
+  } catch (error) {
+    console.error('Failed to establish WebSocket connection:', error);
+    throw error;
+  }
+});
+
+test.after.always('cleanup', async () => {
+  if (wsClient) {
+    wsClient.dispose();
+  }
+  if (testServer) {
+    await testServer.stop();
+  }
+});
+
+// Helper function to collect subscription events
+async function collectSubscriptionEvents(subscription, timeout = 30000) {
+  const events = [];
+  
+  return new Promise((resolve, reject) => {
+    const timeoutId = setTimeout(() => {
+      if (events.length > 0) {
+        resolve(events);
+      } else {
+        reject(new Error('Subscription timed out with no events'));
+      }
+    }, timeout);
+
+    const unsubscribe = wsClient.subscribe(
+      {
+        query: subscription.query,
+        variables: subscription.variables
+      },
+      {
+        next: (event) => {
+          events.push(event);
+          if (event?.data?.requestProgress?.progress === 1) {
+            clearTimeout(timeoutId);
+            unsubscribe();
+            resolve(events);
+          }
+        },
+        error: (error) => {
+          clearTimeout(timeoutId);
+          reject(error);
+        },
+        complete: () => {
+          clearTimeout(timeoutId);
+          resolve(events);
+        }
+      }
+    );
+  });
+}
+
+// Test basic single-step task
+test.serial('sys_entity_agent handles single-step task', async (t) => {
+  t.timeout(30000); // 30 second timeout
+  const response = await testServer.executeOperation({
+    query: `
+      query TestAgentSingleStep(
+        $text: String!, 
+        $chatHistory: [MultiMessage]!
+      ) {
+        sys_entity_agent(
+          text: $text, 
+          chatHistory: $chatHistory,
+          stream: true
+        ) {
+          result
+          contextId
+          tool
+          warnings
+          errors
+        }
+      }
+    `,
+    variables: { 
+      text: 'What is the current time?',
+      chatHistory: [{ 
+        role: "user", 
+        content: ["What is the current time?"] 
+      }]
+    }
+  });
+
+  console.log('Single-step Agent Response:', JSON.stringify(response, null, 2));
+  
+  // Check for successful response
+  t.falsy(response.body?.singleResult?.errors, 'Should not have GraphQL errors');
+  const requestId = response.body?.singleResult?.data?.sys_entity_agent?.result;
+  t.truthy(requestId, 'Should have a requestId in the result field');
+
+  // Collect events
+  const events = await collectSubscriptionEvents({
+    query: `
+      subscription OnRequestProgress($requestId: String!) {
+        requestProgress(requestIds: [$requestId]) {
+          requestId
+          progress
+          data
+          info
+        }
+      }
+    `,
+    variables: { requestId }
+  });
+
+  console.log(`Received ${events.length} events for single-step task`);
+  t.true(events.length > 0, 'Should have received events');
+
+  // Verify we got a completion event
+  const completionEvent = events.find(event => 
+    event.data.requestProgress.progress === 1
+  );
+  t.truthy(completionEvent, 'Should have received a completion event');
+});
+
+// Test multi-step task with tool usage
+test.serial('sys_entity_agent handles multi-step task with tools', async (t) => {
+  t.timeout(60000); // 60 second timeout for multi-step task
+  const response = await testServer.executeOperation({
+    query: `
+      query TestAgentMultiStep(
+        $text: String!, 
+        $chatHistory: [MultiMessage]!
+      ) {
+        sys_entity_agent(
+          text: $text, 
+          chatHistory: $chatHistory,
+          stream: true
+        ) {
+          result
+          contextId
+          tool
+          warnings
+          errors
+        }
+      }
+    `,
+    variables: { 
+      text: 'Research the latest developments in renewable energy and summarize the key trends.',
+      chatHistory: [{ 
+        role: "user", 
+        content: ["Research the latest developments in renewable energy and summarize the key trends."] 
+      }]
+    }
+  });
+
+  console.log('Multi-step Agent Response:', JSON.stringify(response, null, 2));
+  
+  // Check for successful response
+  t.falsy(response.body?.singleResult?.errors, 'Should not have GraphQL errors');
+  const requestId = response.body?.singleResult?.data?.sys_entity_agent?.result;
+  t.truthy(requestId, 'Should have a requestId in the result field');
+
+  // Collect events with a longer timeout since this is a multi-step operation
+  const events = await collectSubscriptionEvents({
+    query: `
+      subscription OnRequestProgress($requestId: String!) {
+        requestProgress(requestIds: [$requestId]) {
+          requestId
+          progress
+          data
+          info
+        }
+      }
+    `,
+    variables: { requestId }
+  }, 60000);
+
+  console.log(`Received ${events.length} events for multi-step task`);
+  t.true(events.length > 0, 'Should have received events');
+
+  // Verify we got a completion event
+  const completionEvent = events.find(event => 
+    event.data.requestProgress.progress === 1
+  );
+  t.truthy(completionEvent, 'Should have received a completion event');
+
+  // Check for tool usage in the events
+  let foundToolUsage = false;
+  for (const event of events) {
+    if (event.data.requestProgress.info) {
+      try {
+        const info = JSON.parse(event.data.requestProgress.info);
+        if (info.toolUsed) {
+          foundToolUsage = true;
+          break;
+        }
+      } catch (e) {
+        // Some info might not be JSON, which is fine
+      }
+    }
+  }
+  t.true(foundToolUsage, 'Should have used tools during execution');
+});