@aj-archipelago/cortex 1.3.35 → 1.3.36

package/tests/call_tools.test.js

@@ -0,0 +1,216 @@
+import test from 'ava';
+import serverFactory from '../index.js';
+
+let testServer;
+
+// List of models to test - comment out models you don't want to test
+const modelsToTest = [
+  'oai-gpt41-mini',
+  'claude-35-sonnet-vertex',
+];
+
+// Add timing data structure
+const modelTimings = {};
+
+// Helper function to track timing
+const trackTiming = (model, startTime) => {
+  const duration = Date.now() - startTime;
+  if (!modelTimings[model]) {
+    modelTimings[model] = [];
+  }
+  modelTimings[model].push(duration);
+};
+
+// Helper function to calculate average timing
+const calculateAverageTiming = (timings) => {
+  return timings.reduce((a, b) => a + b, 0) / timings.length;
+};
+
+// Helper function to print model rankings
+const printModelRankings = () => {
+  const averageTimings = Object.entries(modelTimings).map(([model, timings]) => ({
+    model,
+    avgTime: calculateAverageTiming(timings)
+  }));
+  
+  averageTimings.sort((a, b) => a.avgTime - b.avgTime);
+  
+  console.log('\nModel Performance Rankings:');
+  console.log('-------------------------');
+  averageTimings.forEach((entry, index) => {
+    console.log(`${index + 1}. ${entry.model}: ${Math.round(entry.avgTime)}ms average`);
+  });
+};
+
+// Modify runTestForModels to run tests sequentially
+const runTestForModels = (testName, testFn) => {
+  for (const model of modelsToTest) {
+    test.serial(`${testName}-${model} (sequential)`, async t => {
+      console.log(`\nRunning ${testName} for ${model}...`);
+      const startTime = Date.now();
+
+      try {
+        await testFn(t, model);
+        trackTiming(model, startTime);
+        console.log(`✓ ${model} completed in ${Date.now() - startTime}ms`);
+      } catch (error) {
+        console.log(`✗ ${model} failed after ${Date.now() - startTime}ms`);
+        console.error(error);
+        throw error; // Re-throw to fail the test
+      }
+    });
+  }
+};
+
+test.before(async () => {
+  const { server, startServer } = await serverFactory();
+  startServer && await startServer();
+  testServer = server;
+});
+
+test.after.always('cleanup', async () => {
+  if (testServer) {
+    await testServer.stop();
+  }
+});
+
+// Add after.always hook to print rankings
+test.after.always('print rankings', async () => {
+  printModelRankings();
+});
+
+// Test basic tool calling with a search request
+runTestForModels('call_tools handles search request correctly', async (t, model) => {
+  t.timeout(120000); // 2 minutes timeout for search
+  const response = await testServer.executeOperation({
+    query: `
+      query TestToolCalling($text: String!, $chatHistory: [MultiMessage]!, $model: String) {
+        call_tools(
+          text: $text,
+          chatHistory: $chatHistory,
+          model: $model
+        ) {
+          result
+          contextId
+          tool
+          warnings
+          errors
+        }
+      }
+    `,
+    variables: {
+      text: 'What are the latest developments in renewable energy?',
+      chatHistory: [{
+        role: 'user',
+        content: ['What are the latest developments in renewable energy?']
+      }],
+      model: model
+    }
+  });
+
+  t.is(response.body?.singleResult?.errors, undefined);
+  const result = response.body?.singleResult?.data?.call_tools.result;
+  t.true(result.length > 0, 'Should have a non-empty result');
+});
+
+// Test tool calling with a code execution request
+runTestForModels('call_tools handles code execution request correctly', async (t, model) => {
+  t.timeout(120000); // 2 minutes timeout for code execution
+  const response = await testServer.executeOperation({
+    query: `
+      query TestToolCalling($text: String!, $chatHistory: [MultiMessage]!, $model: String) {
+        call_tools(
+          text: $text,
+          chatHistory: $chatHistory,
+          model: $model
+        ) {
+          result
+          contextId
+          tool
+          warnings
+          errors
+        }
+      }
+    `,
+    variables: {
+      text: 'Write a Python function to calculate fibonacci numbers',
+      chatHistory: [{
+        role: 'user',
+        content: ['Write a Python function to calculate fibonacci numbers']
+      }],
+      model: model
+    }
+  });
+
+  t.is(response.body?.singleResult?.errors, undefined);
+  const result = response.body?.singleResult?.data?.call_tools.result;
+  t.true(result.length > 0, 'Should have a non-empty result');
+});
+
+// Test tool calling with a reasoning request
+runTestForModels('call_tools handles reasoning request correctly', async (t, model) => {
+  t.timeout(120000); // 2 minutes timeout for reasoning
+  const response = await testServer.executeOperation({
+    query: `
+      query TestToolCalling($text: String!, $chatHistory: [MultiMessage]!, $model: String) {
+        call_tools(
+          text: $text,
+          chatHistory: $chatHistory,
+          model: $model
+        ) {
+          result
+          contextId
+          tool
+          warnings
+          errors
+        }
+      }
+    `,
+    variables: {
+      text: 'Explain the implications of quantum computing on cryptography',
+      chatHistory: [{
+        role: 'user',
+        content: ['Explain the implications of quantum computing on cryptography']
+      }],
+      model: model
+    }
+  });
+
+  t.is(response.body?.singleResult?.errors, undefined);
+  const result = response.body?.singleResult?.data?.call_tools.result;
+  t.true(result.length > 0, 'Should have a non-empty result');
+});
+
+// Test tool calling with a document request
+runTestForModels('call_tools handles document request correctly', async (t, model) => {
+  t.timeout(120000); // 2 minutes timeout for document processing
+  const response = await testServer.executeOperation({
+    query: `
+      query TestToolCalling($text: String!, $chatHistory: [MultiMessage]!, $model: String) {
+        call_tools(
+          text: $text,
+          chatHistory: $chatHistory,
+          model: $model
+        ) {
+          result
+          contextId
+          tool
+          warnings
+          errors
+        }
+      }
+    `,
+    variables: {
+      text: 'Summarize the key points from my document about project management',
+      chatHistory: [{
+        role: 'user',
+        content: ['Summarize the key points from my document about project management']
+      }],
+      model: model
+    }
+  });
+
+  t.is(response.body?.singleResult?.errors, undefined);
+  const result = response.body?.singleResult?.data?.call_tools.result;
+  t.true(result.length > 0, 'Should have a non-empty result');
+}); 

package/tests/claude3VertexToolConversion.test.js

@@ -408,4 +408,82 @@ test('Combining existing tools block with generated tools', async (t) => {
     t.truthy(result.tools, 'Tools should be defined');
     t.is(result.tools.length, 2, 'Should have 2 tools');
     t.deepEqual(result.tools.map(t => t.name).sort(), ['get_weather', 'memory_lookup']);
+});
+
+// Test preventing duplicate tool definitions
+test('Prevent duplicate tool definitions', async (t) => {
+    const plugin = createPlugin();
+    const prompt = mockPathwayResolverMessages.pathway.prompt;
+    
+    const parameters = {
+        tools: [
+            {
+                type: 'function',
+                function: {
+                    name: 'memory_lookup',
+                    description: 'Look up information in memory',
+                    parameters: {
+                        type: 'object',
+                        properties: {
+                            query: {
+                                type: 'string',
+                                description: 'The search query'
+                            }
+                        },
+                        required: ['query']
+                    }
+                }
+            }
+        ]
+    };
+
+    const messages = [
+        {
+            role: 'system',
+            content: 'You are a helpful assistant'
+        },
+        {
+            role: 'user',
+            content: 'What\'s in my memory?'
+        },
+        {
+            role: 'assistant',
+            content: [
+                {
+                    type: 'tool_use',
+                    id: 'tool_1',
+                    name: 'memory_lookup',
+                    input: { query: 'search memory' }
+                }
+            ]
+        }
+    ];
+
+    // Set up the mock prompt with messages
+    prompt.messages = messages;
+
+    const cortexRequest = { messages };
+    const result = await plugin.getRequestParameters('test', parameters, prompt, cortexRequest);
+    
+    // Check that we only have one memory_lookup tool definition
+    t.truthy(result.tools, 'Tools should be defined');
+    t.is(result.tools.length, 1, 'Should have exactly 1 tool');
+    t.is(result.tools[0].name, 'memory_lookup', 'Tool should be memory_lookup');
+    t.is(result.tools[0].description, 'Look up information in memory', 'Should preserve original tool description');
+    
+    // Verify the tool_use call is still properly converted
+    t.truthy(result.messages, 'Messages should be defined');
+    t.is(result.messages.length, 1, 'Should have 1 message after conversion');
+    
+    // Check the converted message
+    const message = result.messages[0];
+    t.is(message.role, 'assistant', 'Message should be from assistant');
+    t.truthy(message.content, 'Message should have content');
+    t.is(message.content.length, 1, 'Message should have one content item');
+    t.deepEqual(message.content[0], {
+        type: 'tool_use',
+        id: 'tool_1',
+        name: 'memory_lookup',
+        input: { query: 'search memory' }
+    });
 });

package/tests/mocks.js

@@ -68,9 +68,17 @@ export const mockConfig = {
 
   export const mockPathwayResolverMessages = {
     model: {
-      name: 'testModel',
-      url: 'https://api.example.com/testModel',
-      type: 'OPENAI-COMPLETION',
+        name: 'testModel',
+        type: 'OPENAI-CHAT',
+        url: 'https://api.openai.com/v1/chat/completions',
+        endpoints: [{
+            name: 'Test Endpoint',
+            url: 'https://api.openai.com/v1/chat/completions',
+            headers: {
+                'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`,
+                'Content-Type': 'application/json'
+            }
+        }]
     },
     modelName: 'testModel',
     pathway: mockPathwayMessages,

package/tests/multimodal_conversion.test.js

@@ -209,7 +209,7 @@ test('Unsupported mime type conversion', async (t) => {
 
     t.is(modifiedMessages[0].content.length, 2);
     t.is(modifiedMessages[0].content[0].text, 'Can you analyze this PDF?');
-    t.is(modifiedMessages[0].content[1].text, 'Image skipped: unsupported format');
+    t.true(modifiedMessages[0].content[1].text.includes('image_url'));
 });
 
 // Test pathological cases

package/tests/openAiToolPlugin.test.js

@@ -0,0 +1,242 @@
+import test from 'ava';
+import OpenAIVisionPlugin from '../server/plugins/openAiVisionPlugin.js';
+import { mockPathwayResolverMessages } from './mocks.js';
+import { config } from '../config.js';
+
+const { pathway, modelName, model } = mockPathwayResolverMessages;
+
+// Helper function to create a plugin instance
+const createPlugin = () => {
+    const plugin = new OpenAIVisionPlugin(pathway, {
+        name: 'test-model',
+        type: 'OPENAI-VISION'
+    });
+    return plugin;
+};
+
+// Test OpenAI tools block conversion
+test('OpenAI tools block conversion', async (t) => {
+    const plugin = createPlugin();
+    const prompt = mockPathwayResolverMessages.pathway.prompt;
+    
+    const parameters = {
+        tools: [
+            {
+                type: 'function',
+                function: {
+                    name: 'get_weather',
+                    description: 'Get current temperature for a given location.',
+                    parameters: {
+                        type: 'object',
+                        properties: {
+                            location: {
+                                type: 'string',
+                                description: 'City and country e.g. Bogotá, Colombia'
+                            }
+                        },
+                        required: ['location'],
+                        additionalProperties: false
+                    }
+                }
+            }
+        ]
+    };
+
+    const cortexRequest = { tools: parameters.tools };
+    const result = await plugin.getRequestParameters('test', parameters, prompt, cortexRequest);
+    
+    t.deepEqual(result.tools, parameters.tools);
+});
+
+// Test tool call response handling
+test('Tool call response handling', async (t) => {
+    const plugin = createPlugin();
+    
+    const responseData = {
+        choices: [{
+            message: {
+                role: 'assistant',
+                content: 'I will check the weather for you.',
+                tool_calls: [{
+                    id: 'call_123',
+                    type: 'function',
+                    function: {
+                        name: 'get_weather',
+                        arguments: '{"location": "Bogotá, Colombia"}'
+                    }
+                }]
+            }
+        }]
+    };
+
+    const result = plugin.parseResponse(responseData);
+    
+    t.deepEqual(result, {
+        role: 'assistant',
+        content: 'I will check the weather for you.',
+        tool_calls: [{
+            id: 'call_123',
+            type: 'function',
+            function: {
+                name: 'get_weather',
+                arguments: '{"location": "Bogotá, Colombia"}'
+            }
+        }]
+    });
+});
+
+// Test tool result message handling
+test('Tool result message handling', async (t) => {
+    const plugin = createPlugin();
+    const prompt = mockPathwayResolverMessages.pathway.prompt;
+    
+    const messages = [
+        {
+            role: 'assistant',
+            content: 'I will check the weather for you.',
+            tool_calls: [{
+                id: 'call_123',
+                type: 'function',
+                function: {
+                    name: 'get_weather',
+                    arguments: '{"location": "Bogotá, Colombia"}'
+                }
+            }]
+        },
+        {
+            role: 'tool',
+            content: 'The weather in Bogotá is 18°C and sunny.',
+            tool_call_id: 'call_123'
+        }
+    ];
+
+    const result = await plugin.tryParseMessages(messages);
+    
+    t.deepEqual(result, messages);
+});
+
+// Test mixed content with tools and images
+test('Mixed content with tools and images', async (t) => {
+    const plugin = createPlugin();
+    const prompt = mockPathwayResolverMessages.pathway.prompt;
+    
+    // Mock the validateImageUrl method to always return true
+    plugin.validateImageUrl = async () => true;
+    
+    const messages = [
+        {
+            role: 'user',
+            content: [
+                { type: 'text', text: 'What\'s the weather in this image?' },
+                { type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } }
+            ]
+        },
+        {
+            role: 'assistant',
+            content: 'I will analyze the image and check the weather.',
+            tool_calls: [{
+                id: 'call_123',
+                type: 'function',
+                function: {
+                    name: 'get_weather',
+                    arguments: '{"location": "Bogotá, Colombia"}'
+                }
+            }]
+        }
+    ];
+
+    const result = await plugin.tryParseMessages(messages);
+    
+    t.is(result[0].role, 'user');
+    t.is(result[0].content[0].type, 'text');
+    t.is(result[0].content[1].type, 'image_url');
+    t.is(result[1].role, 'assistant');
+    t.truthy(result[1].tool_calls);
+});
+
+// Test error handling in tool calls
+test('Error handling in tool calls', async (t) => {
+    const plugin = createPlugin();
+    
+    const responseData = {
+        choices: [{
+            message: {
+                role: 'assistant',
+                content: 'I will check the weather for you.',
+                tool_calls: [{
+                    id: 'call_123',
+                    type: 'function',
+                    function: {
+                        name: 'get_weather',
+                        arguments: 'invalid json'
+                    }
+                }]
+            }
+        }]
+    };
+
+    const result = plugin.parseResponse(responseData);
+    
+    t.deepEqual(result, {
+        role: 'assistant',
+        content: 'I will check the weather for you.',
+        tool_calls: [{
+            id: 'call_123',
+            type: 'function',
+            function: {
+                name: 'get_weather',
+                arguments: 'invalid json'
+            }
+        }]
+    });
+});
+
+// Test multiple tool calls in sequence
+test('Multiple tool calls in sequence', async (t) => {
+    const plugin = createPlugin();
+    const prompt = mockPathwayResolverMessages.pathway.prompt;
+    
+    const messages = [
+        {
+            role: 'assistant',
+            content: 'I will check multiple things for you.',
+            tool_calls: [
+                {
+                    id: 'call_123',
+                    type: 'function',
+                    function: {
+                        name: 'get_weather',
+                        arguments: '{"location": "Bogotá, Colombia"}'
+                    }
+                },
+                {
+                    id: 'call_124',
+                    type: 'function',
+                    function: {
+                        name: 'get_time',
+                        arguments: '{"location": "Bogotá, Colombia"}'
+                    }
+                }
+            ]
+        },
+        {
+            role: 'tool',
+            content: 'The weather in Bogotá is 18°C and sunny.',
+            tool_call_id: 'call_123'
+        },
+        {
+            role: 'tool',
+            content: 'The current time in Bogotá is 14:30.',
+            tool_call_id: 'call_124'
+        }
+    ];
+
+    const result = await plugin.tryParseMessages(messages);
+    
+    t.is(result.length, 3);
+    t.is(result[0].role, 'assistant');
+    t.is(result[0].tool_calls.length, 2);
+    t.is(result[1].role, 'tool');
+    t.is(result[2].role, 'tool');
+});
+

package/pathways/test_palm_chat.js

@@ -1,31 +0,0 @@
-//test_palm_chat.mjs
-// Test for handling of prompts in the PaLM chat format for Cortex
-
-import { Prompt } from '../server/prompt.js';
-
-// Description: Have a chat with a bot that uses context to understand the conversation
-export default {
-    prompt:
-        [
-            new Prompt({ 
-                context: "Instructions:\nYou an AI entity working a global media network. You are truthful, kind, and helpful. Your expertise includes journalism, journalistic ethics, researching and composing documents, and technology. You know the current date and time - it is {{now}}.",
-                examples: [
-                    { 
-                       input: {"content": "What is your expertise?"},
-                       output: {"content": "I am an expert in journalism and journalistic ethics."}
-                    }],
-                messages: [
-                {"author": "user", "content": "Hi how are you today?"},
-                {"author": "assistant", "content": "I am doing well. How are you?"},
-                {"author": "user", "content": "I am doing well. What is your name?"},
-                {"author": "assistant", "content": "My name is Hula. What is your name?"},
-                {"author": "user", "content": "My name is Bob. What is your expertise?"},
-            ]}),
-        ],
-    inputParameters: {
-        chatHistory: [],
-        contextId: ``,
-    },
-    model: 'palm-chat',
-    useInputChunking: false,
-}