@aj-archipelago/cortex 1.3.21 → 1.3.23

package/tests/multimodal_conversion.test.js

@@ -271,12 +271,18 @@ test('Pathological cases', async (t) => {
 
     t.is(geminiSystem15.parts[0].text, 'You are a helpful assistant.');
     t.is(geminiSystem15.parts[1].text, 'You are also very knowledgeable.');
+
     t.is(geminiMessages15.length, 3);
+    // First user message combines "Hello" and "Another greeting"
     t.is(geminiMessages15[0].role, 'user');
     t.is(geminiMessages15[0].parts[0].text, 'Hello');
     t.is(geminiMessages15[0].parts[1].text, 'Another greeting');
+    
+    // Assistant message "Hi there!"
     t.is(geminiMessages15[1].role, 'assistant');
     t.is(geminiMessages15[1].parts[0].text, 'Hi there!');
+     
+    // Final user message combines "How are you?", image content, and "Another question"
     t.is(geminiMessages15[2].role, 'user');
     t.is(geminiMessages15[2].parts[0].text, 'How are you?');
     t.is(geminiMessages15[2].parts[1].text, 'What\'s this?');
@@ -310,6 +316,79 @@ test('Empty message array', async (t) => {
     t.is(geminiMessages15.length, 0);
 });
 
+// Test simple string array content
+test('Simple string array content', async (t) => {
+    const { gemini15 } = createPlugins();
+    
+    const messages = [
+        { role: 'user', content: "Initial message" },
+        { role: 'assistant', content: [
+            "\"Searchin' for my lost shaker of salt...\"\n",
+        ]},
+        { role: 'user', content: [
+            "Here's another simple string in an array",
+        ]}
+    ];
+
+    const { modifiedMessages } = gemini15.convertMessagesToGemini(messages);
+
+    t.is(modifiedMessages.length, 3);
+    t.is(modifiedMessages[0].role, 'user');
+    t.is(modifiedMessages[0].parts.length, 1);
+    t.is(modifiedMessages[0].parts[0].text, "Initial message");
+    t.is(modifiedMessages[1].role, 'assistant');
+    t.is(modifiedMessages[1].parts.length, 1);
+    t.is(modifiedMessages[1].parts[0].text, "\"Searchin' for my lost shaker of salt...\"\n");
+    t.is(modifiedMessages[2].role, 'user');
+    t.is(modifiedMessages[2].parts.length, 1);
+    t.is(modifiedMessages[2].parts[0].text, "Here's another simple string in an array");
+});
+
+// Test string-encoded multimodal content
+test('String-encoded multimodal content', async (t) => {
+    const { gemini15 } = createPlugins();
+    
+    const messages = [
+        { role: 'user', content: [
+            JSON.stringify({
+                type: 'text',
+                text: 'What is in this image?'
+            }),
+            JSON.stringify({
+                type: 'image_url',
+                image_url: { url: 'gs://my-bucket/image.jpg' }
+            })
+        ]},
+        { role: 'assistant', content: [
+            JSON.stringify({
+                type: 'text',
+                text: 'I see a cat.'
+            })
+        ]},
+        { role: 'user', content: [
+            JSON.stringify({
+                type: 'text',
+                text: 'Is it a big cat?'
+            })
+        ]}
+    ];
+
+    const { modifiedMessages } = gemini15.convertMessagesToGemini(messages);
+
+    t.is(modifiedMessages.length, 3);
+    t.is(modifiedMessages[0].role, 'user');
+    t.is(modifiedMessages[0].parts.length, 2);
+    t.is(modifiedMessages[0].parts[0].text, 'What is in this image?');
+    t.true('fileData' in modifiedMessages[0].parts[1]);
+    t.is(modifiedMessages[0].parts[1].fileData.fileUri, 'gs://my-bucket/image.jpg');
+    t.is(modifiedMessages[1].role, 'assistant');
+    t.is(modifiedMessages[1].parts.length, 1);
+    t.is(modifiedMessages[1].parts[0].text, 'I see a cat.');
+    t.is(modifiedMessages[2].role, 'user');
+    t.is(modifiedMessages[2].parts.length, 1);
+    t.is(modifiedMessages[2].parts[0].text, 'Is it a big cat?');
+});
+
 // Test messages with only system messages
 test('Only system messages', async (t) => {
     const { openai, claude, gemini, gemini15 } = createPlugins();
@@ -417,3 +496,93 @@ test('Gemini 1.5 image URL edge cases', t => {
     // Verify we only have one part (the text)
     t.is(modifiedMessages[0].parts.length, 1, 'Should only have the text part');
 });
+
+// Test multiple images in single message for Claude
+test('Multiple images in single Claude message', async (t) => {
+    const { claude } = createPlugins();
+    
+    const multiImageMessage = [
+        { role: 'user', content: [
+            { type: 'text', text: 'Compare these images:' },
+            { type: 'image_url', image_url: { url: sampleBase64Image } },
+            { type: 'text', text: 'with this one:' },
+            { type: 'image_url', image_url: { url: sampleBase64Image } },
+            { type: 'image_url', gcs: 'gs://cortex-bucket/image.jpg' }
+        ]}
+    ];
+
+    const { modifiedMessages } = await claude.convertMessagesToClaudeVertex(multiImageMessage);
+
+    t.is(modifiedMessages.length, 1);
+    t.is(modifiedMessages[0].role, 'user');
+    t.is(modifiedMessages[0].content.length, 4);
+    t.is(modifiedMessages[0].content[0].text, 'Compare these images:');
+    t.true(modifiedMessages[0].content[1].source.type === 'base64');
+    t.is(modifiedMessages[0].content[2].text, 'with this one:');
+    t.true(modifiedMessages[0].content[3].source.type === 'base64');
+});
+
+// Test conversation history with mixed image types
+test('Conversation history with mixed image types', async (t) => {
+    const { claude, gemini15 } = createPlugins();
+    
+    const conversationHistory = [
+        { role: 'system', content: 'You are a visual analysis assistant.' },
+        { role: 'user', content: [
+            { type: 'text', text: 'What\'s in this image?' },
+            { type: 'image_url', image_url: { url: sampleBase64Image } }
+        ]},
+        { role: 'assistant', content: 'I see a landscape.' },
+        { role: 'user', content: [
+            { type: 'text', text: 'Compare it with this:' },
+            { type: 'image_url', gcs: 'gs://cortex-bucket/image2.jpg' }
+        ]},
+        { role: 'assistant', content: 'The second image shows a different scene.' },
+        { role: 'user', content: 'Which one do you prefer?' }
+    ];
+
+    // Test Claude conversion
+    const { system: claudeSystem, modifiedMessages: claudeMessages } = await claude.convertMessagesToClaudeVertex(conversationHistory);
+
+    t.is(claudeSystem, 'You are a visual analysis assistant.');
+    t.is(claudeMessages.length, 5);
+    t.is(claudeMessages[1].content[0].text, 'I see a landscape.');
+    t.is(claudeMessages[3].content[0].text, 'The second image shows a different scene.');
+    t.is(claudeMessages[4].content[0].text, 'Which one do you prefer?');
+
+    // Test Gemini 1.5 conversion
+    const { system: geminiSystem15, modifiedMessages: geminiMessages15 } = gemini15.convertMessagesToGemini(conversationHistory);
+
+    t.is(geminiSystem15.parts[0].text, 'You are a visual analysis assistant.');
+    t.is(geminiMessages15.length, 5);
+    t.true('inlineData' in geminiMessages15[0].parts[1]);
+    t.is(geminiMessages15[1].parts[0].text, 'I see a landscape.');
+    t.true('fileData' in geminiMessages15[2].parts[1]);
+    t.is(geminiMessages15[2].parts[1].fileData.fileUri, 'gs://cortex-bucket/image2.jpg');
+    t.is(geminiMessages15[3].parts[0].text, 'The second image shows a different scene.');
+    t.is(geminiMessages15[4].parts[0].text, 'Which one do you prefer?');
+});
+
+// Test handling of large images
+test('Large image handling', async (t) => {
+    const { claude, gemini15 } = createPlugins();
+    
+    // Create a large base64 string (>10MB)
+    const largeSampleImage = 'data:image/jpeg;base64,' + 'A'.repeat(10 * 1024 * 1024);
+    
+    const largeImageMessage = [
+        { role: 'user', content: [
+            { type: 'text', text: 'Check this large image:' },
+            { type: 'image_url', image_url: { url: largeSampleImage } }
+        ]}
+    ];
+
+    // Both Claude and Gemini should handle or reject oversized images gracefully
+    const { modifiedMessages: claudeMessages } = await claude.convertMessagesToClaudeVertex(largeImageMessage);
+    const { modifiedMessages: geminiMessages } = gemini15.convertMessagesToGemini(largeImageMessage);
+
+    // Verify both models handle the oversized image appropriately
+    // (The exact behavior - rejection vs. compression - should match the model's specifications)
+    t.is(claudeMessages[0].content[0].text, 'Check this large image:');
+    t.is(geminiMessages[0].parts[0].text, 'Check this large image:');
+});

package/tests/openai_api.test.js

@@ -60,6 +60,36 @@ test('POST /chat/completions', async (t) => {
   t.true(Array.isArray(response.body.choices));
 });
 
+test('POST /chat/completions with multimodal content', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'claude-3.5-sonnet',
+      messages: [{
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What do you see in this image?'
+          },
+          {
+            type: 'image',
+            image_url: {
+              url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDABQODxIPDRQSEBIXFRQdHx4eHRoaHSQrJyEwPENBMDQ4NDQ0QUJCSkNLS0tNSkpQUFFQR1BTYWNgY2FQYWFQYWj/2wBDARUXFyAeIBohHh4oIiE2LCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAoDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAb/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdABmX/9k='
+            }
+          }
+        ]
+      }],
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].message.content);
+});
+
 async function connectToSSEEndpoint(url, endpoint, payload, t, customAssertions) {
     return new Promise(async (resolve, reject) => {
         try {
@@ -143,5 +173,307 @@ test('POST SSE: /v1/chat/completions should send a series of events and a [DONE]
     };
     
     await connectToSSEEndpoint(url, '/chat/completions', payload, t, chatCompletionsAssertions);
+});
+
+test('POST SSE: /v1/chat/completions with multimodal content should send a series of events and a [DONE] event', async (t) => {
+    const payload = {
+        model: 'claude-3.5-sonnet',
+        messages: [{
+          role: 'user',
+          content: [
+            {
+              type: 'text',
+              text: 'What do you see in this image?'
+            },
+            {
+              type: 'image',
+              image_url: {
+                url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDABQODxIPDRQSEBIXFRQdHx4eHRoaHSQrJyEwPENBMDQ4NDQ0QUJCSkNLS0tNSkpQUFFQR1BTYWNgY2FQYWFQYWj/2wBDARUXFyAeIBohHh4oIiE2LCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAoDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAb/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdABmX/9k='
+              }
+            }
+          ]
+        }],
+        stream: true,
+    };
+    
+    const url = `http://localhost:${process.env.CORTEX_PORT}/v1`;
+    
+    const multimodalChatCompletionsAssertions = (t, messageJson) => {
+        t.truthy(messageJson.id);
+        t.is(messageJson.object, 'chat.completion.chunk');
+        t.truthy(messageJson.choices[0].delta);
+        if (messageJson.choices[0].finish_reason === 'stop') {
+          t.truthy(messageJson.choices[0].delta);
+        }
+    };
+    
+    await connectToSSEEndpoint(url, '/chat/completions', payload, t, multimodalChatCompletionsAssertions);
+});  
+
+test('POST /chat/completions should handle multimodal content for non-multimodal model', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'gpt-3.5-turbo',
+      messages: [{
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What do you see in this image?'
+          },
+          {
+            type: 'image',
+            image_url: {
+              url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD...'
+            }
+          }
+        ]
+      }],
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].message.content);
+});
+
+test('POST SSE: /v1/chat/completions should handle streaming multimodal content for non-multimodal model', async (t) => {
+  const payload = {
+    model: 'gpt-3.5-turbo',
+    messages: [{
+      role: 'user',
+      content: [
+        {
+          type: 'text',
+          text: 'What do you see in this image?'
+        },
+        {
+          type: 'image',
+          image_url: {
+            url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD...'
+          }
+        }
+      ]
+    }],
+    stream: true,
+  };
+
+  const streamingAssertions = (t, messageJson) => {
+    t.truthy(messageJson.id);
+    t.is(messageJson.object, 'chat.completion.chunk');
+    t.truthy(messageJson.choices[0].delta);
+    if (messageJson.choices[0].finish_reason === 'stop') {
+      t.truthy(messageJson.choices[0].delta);
+    }
+  };
+
+  await connectToSSEEndpoint(
+    `http://localhost:${process.env.CORTEX_PORT}/v1`,
+    '/chat/completions',
+    payload,
+    t,
+    streamingAssertions
+  );
+});
+
+test('POST /chat/completions should handle malformed multimodal content', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'claude-3.5-sonnet',
+      messages: [{
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            // Missing text field
+          },
+          {
+            type: 'image',
+            // Missing image_url
+          }
+        ]
+      }],
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].message.content);
+});
+
+test('POST /chat/completions should handle invalid image data', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'claude-3.5-sonnet',
+      messages: [{
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: 'What do you see in this image?'
+          },
+          {
+            type: 'image',
+            image_url: {
+              url: 'not-a-valid-base64-image'
+            }
+          }
+        ]
+      }],
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].message.content);
+});  
+
+test('POST /completions should handle model parameters', async (t) => {
+  const response = await got.post(`${API_BASE}/completions`, {
+    json: {
+      model: 'gpt-3.5-turbo',
+      prompt: 'Say this is a test',
+      temperature: 0.7,
+      max_tokens: 100,
+      top_p: 1,
+      frequency_penalty: 0,
+      presence_penalty: 0,
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'text_completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].text);
+});
+
+test('POST /chat/completions should handle function calling', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'gpt-3.5-turbo',
+      messages: [{ role: 'user', content: 'What is the weather in Boston?' }],
+      functions: [{
+        name: 'get_weather',
+        description: 'Get the current weather in a given location',
+        parameters: {
+          type: 'object',
+          properties: {
+            location: {
+              type: 'string',
+              description: 'The city and state, e.g. San Francisco, CA'
+            },
+            unit: {
+              type: 'string',
+              enum: ['celsius', 'fahrenheit']
+            }
+          },
+          required: ['location']
+        }
+      }],
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  const choice = response.body.choices[0];
+  t.true(['function_call', 'stop'].includes(choice.finish_reason));
+  if (choice.finish_reason === 'function_call') {
+    t.truthy(choice.message.function_call);
+    t.truthy(choice.message.function_call.name);
+    t.truthy(choice.message.function_call.arguments);
+  }
+});
+
+test('POST /chat/completions should validate response format', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'gpt-3.5-turbo',
+      messages: [{ role: 'user', content: 'Hello!' }],
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.id);
+  t.truthy(response.body.created);
+  t.truthy(response.body.model);
+  
+  const choice = response.body.choices[0];
+  t.is(typeof choice.index, 'number');
+  t.truthy(choice.message);
+  t.truthy(choice.message.role);
+  t.truthy(choice.message.content);
+  t.truthy(choice.finish_reason);
+});
+
+test('POST /chat/completions should handle system messages', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'gpt-3.5-turbo',
+      messages: [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'Hello!' }
+      ],
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].message.content);
+});
+
+test('POST /chat/completions should handle errors gracefully', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      // Missing required model field
+      messages: [{ role: 'user', content: 'Hello!' }],
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].message.content);
+});
+
+test('POST /chat/completions should handle token limits', async (t) => {
+  const response = await got.post(`${API_BASE}/chat/completions`, {
+    json: {
+      model: 'gpt-3.5-turbo',
+      messages: [{ 
+        role: 'user', 
+        content: 'Hello!'.repeat(5000) // Very long message
+      }],
+      max_tokens: 100,
+      stream: false,
+    },
+    responseType: 'json',
+  });
+
+  t.is(response.statusCode, 200);
+  t.is(response.body.object, 'chat.completion');
+  t.true(Array.isArray(response.body.choices));
+  t.truthy(response.body.choices[0].message.content);
 });