@aj-archipelago/cortex 1.4.31 → 1.4.33

package/tests/integration/clientToolCallbacks.test.js

@@ -0,0 +1,161 @@
+// Test for client tool callbacks with Redis pub/sub
+import { describe, it, expect, beforeAll, afterAll } from '@jest/globals';
+import { 
+    waitForClientToolResult, 
+    resolveClientToolCallback,
+    getPendingCallbackCount 
+} from '../../server/clientToolCallbacks.js';
+
+describe('Client Tool Callbacks - Multi-Instance Support', () => {
+    const mockRequestId = 'test-request-123';
+    
+    afterAll(async () => {
+        // Give Redis time to clean up connections
+        await new Promise(resolve => setTimeout(resolve, 100));
+    });
+
+    it('should register and resolve a callback locally', async () => {
+        const toolCallbackId = 'test-callback-1';
+        
+        // Start waiting for result
+        const waitPromise = waitForClientToolResult(toolCallbackId, mockRequestId, 5000);
+        
+        // Verify callback is registered
+        expect(getPendingCallbackCount()).toBeGreaterThan(0);
+        
+        // Simulate client submitting result
+        const testResult = {
+            success: true,
+            data: { message: 'Test completed' },
+            error: null
+        };
+        
+        // Resolve the callback (this will publish to Redis if available, or resolve locally)
+        const resolved = await resolveClientToolCallback(toolCallbackId, testResult);
+        expect(resolved).toBe(true);
+        
+        // Wait for the result
+        const result = await waitPromise;
+        
+        expect(result).toEqual(testResult);
+        expect(result.success).toBe(true);
+        expect(result.data.message).toBe('Test completed');
+    });
+
+    it('should timeout if no result is received', async () => {
+        const toolCallbackId = 'test-callback-timeout';
+        
+        // Start waiting with a short timeout
+        const waitPromise = waitForClientToolResult(toolCallbackId, mockRequestId, 100);
+        
+        // Don't resolve - let it timeout
+        await expect(waitPromise).rejects.toThrow('Client tool execution timeout');
+    }, 10000);
+
+    it('should handle callback with error result', async () => {
+        const toolCallbackId = 'test-callback-error';
+        
+        // Start waiting for result
+        const waitPromise = waitForClientToolResult(toolCallbackId, mockRequestId, 5000);
+        
+        // Simulate client submitting error result
+        const errorResult = {
+            success: false,
+            data: null,
+            error: 'Tool execution failed'
+        };
+        
+        // Resolve with error
+        await resolveClientToolCallback(toolCallbackId, errorResult);
+        
+        // Wait for the result
+        const result = await waitPromise;
+        
+        expect(result.success).toBe(false);
+        expect(result.error).toBe('Tool execution failed');
+    });
+
+    it('should handle multiple concurrent callbacks', async () => {
+        const callbacks = [];
+        const numCallbacks = 5;
+        
+        // Register multiple callbacks
+        for (let i = 0; i < numCallbacks; i++) {
+            const callbackId = `concurrent-callback-${i}`;
+            const promise = waitForClientToolResult(callbackId, mockRequestId, 5000);
+            callbacks.push({ id: callbackId, promise });
+        }
+        
+        // Resolve all callbacks
+        for (let i = 0; i < numCallbacks; i++) {
+            const result = {
+                success: true,
+                data: { index: i, message: `Result ${i}` },
+                error: null
+            };
+            await resolveClientToolCallback(callbacks[i].id, result);
+        }
+        
+        // Wait for all results
+        const results = await Promise.all(callbacks.map(cb => cb.promise));
+        
+        // Verify all results
+        expect(results).toHaveLength(numCallbacks);
+        results.forEach((result, index) => {
+            expect(result.success).toBe(true);
+            expect(result.data.index).toBe(index);
+        });
+    });
+
+    it('should return false when resolving non-existent callback', async () => {
+        const nonExistentId = 'non-existent-callback-id';
+        
+        const result = {
+            success: true,
+            data: { message: 'Test' },
+            error: null
+        };
+        
+        // This should publish to Redis (if available) or return false locally
+        // Either way, it should not throw an error
+        const resolved = await resolveClientToolCallback(nonExistentId, result);
+        
+        // In Redis mode, this returns true (published)
+        // In local mode, this returns false (not found)
+        expect(typeof resolved).toBe('boolean');
+    });
+});
+
+describe('Client Tool Callbacks - Performance', () => {
+    it('should handle rapid callback resolution', async () => {
+        const start = Date.now();
+        const numCallbacks = 100;
+        const callbacks = [];
+        
+        // Register many callbacks rapidly
+        for (let i = 0; i < numCallbacks; i++) {
+            const callbackId = `perf-callback-${i}`;
+            const promise = waitForClientToolResult(callbackId, 'perf-test', 5000);
+            callbacks.push({ id: callbackId, promise });
+        }
+        
+        // Resolve all callbacks rapidly
+        for (let i = 0; i < numCallbacks; i++) {
+            await resolveClientToolCallback(callbacks[i].id, {
+                success: true,
+                data: { index: i },
+                error: null
+            });
+        }
+        
+        // Wait for all
+        const results = await Promise.all(callbacks.map(cb => cb.promise));
+        const duration = Date.now() - start;
+        
+        expect(results).toHaveLength(numCallbacks);
+        expect(duration).toBeLessThan(10000); // Should complete in under 10 seconds
+        
+        console.log(`Performance test: ${numCallbacks} callbacks resolved in ${duration}ms (${(duration/numCallbacks).toFixed(2)}ms avg per callback)`);
+    }, 15000);
+});
+

package/tests/integration/features/tools/fileCollection.test.js

@@ -1086,7 +1086,7 @@ test('File collection: syncAndStripFilesFromChatHistory only strips collection f
         ];
         
         // Process chat history
-        const { chatHistory: processed } = await syncAndStripFilesFromChatHistory(chatHistory, createAgentContext(contextId));
+        const { chatHistory: processed, availableFiles } = await syncAndStripFilesFromChatHistory(chatHistory, createAgentContext(contextId));
         
         // Verify only collection file was stripped
         const content = processed[0].content;

package/tests/integration/rest/vendors/claude_anthropic_direct.test.js

@@ -0,0 +1,197 @@
+/**
+ * E2E integration test for direct Anthropic API access (not via Vertex AI)
+ * Tests the CLAUDE-ANTHROPIC plugin type with Claude models
+ * 
+ * Run with: npm test -- tests/integration/rest/vendors/claude_anthropic_direct.test.js
+ */
+import test from 'ava';
+import serverFactory from '../../../../index.js';
+import got from 'got';
+import { collectSSEChunks, assertOAIChatChunkBasics, assertAnyContentDelta } from '../../../helpers/sseAssert.js';
+
+let testServer;
+
+test.before(async () => {
+  process.env.CORTEX_ENABLE_REST = 'true';
+  const { server, startServer } = await serverFactory();
+  startServer && await startServer();
+  testServer = server;
+});
+
+test.after.always('cleanup', async () => {
+  if (testServer) await testServer.stop();
+});
+
+// Test: Basic streaming chat completion with Claude 4.5 Sonnet via direct Anthropic API
+test('Claude 4.5 Sonnet (direct Anthropic) SSE streaming chat', async (t) => {
+  const baseUrl = `http://localhost:${process.env.CORTEX_PORT}/v1`;
+
+  // Use the direct Anthropic model (claude-45-sonnet)
+  const model = 'claude-45-sonnet';
+  
+  // Verify the model is available
+  try {
+    const res = await got(`${baseUrl}/models`, { responseType: 'json' });
+    const ids = (res.body?.data || []).map(m => m.id);
+    if (!ids.includes(model)) {
+      t.pass(`Skipping - model ${model} not configured`);
+      return;
+    }
+  } catch (err) {
+    t.fail(`Failed to get models: ${err.message}`);
+    return;
+  }
+
+  const payload = {
+    model,
+    messages: [{ role: 'user', content: 'Say "Hello from Anthropic direct API!" and nothing else.' }],
+    stream: true,
+  };
+
+  const chunks = await collectSSEChunks(baseUrl, '/chat/completions', payload);
+  t.true(chunks.length > 0, 'Should receive SSE chunks');
+  chunks.forEach(ch => assertOAIChatChunkBasics(t, ch));
+  t.true(assertAnyContentDelta(chunks), 'Should have content delta in chunks');
+  
+  // Log the full response for debugging
+  const fullContent = chunks
+    .map(c => c?.choices?.[0]?.delta?.content || '')
+    .join('');
+  t.log(`Response: ${fullContent}`);
+});
+
+// Test: Non-streaming chat completion with Claude 4.5 Sonnet via direct Anthropic API
+test('Claude 4.5 Sonnet (direct Anthropic) non-streaming chat', async (t) => {
+  const baseUrl = `http://localhost:${process.env.CORTEX_PORT}/v1`;
+  const model = 'claude-45-sonnet';
+
+  // Verify the model is available
+  try {
+    const res = await got(`${baseUrl}/models`, { responseType: 'json' });
+    const ids = (res.body?.data || []).map(m => m.id);
+    if (!ids.includes(model)) {
+      t.pass(`Skipping - model ${model} not configured`);
+      return;
+    }
+  } catch (err) {
+    t.fail(`Failed to get models: ${err.message}`);
+    return;
+  }
+
+  const payload = {
+    model,
+    messages: [{ role: 'user', content: 'What is 2 + 2? Reply with just the number.' }],
+    stream: false,
+  };
+
+  try {
+    const response = await got.post(`${baseUrl}/chat/completions`, {
+      json: payload,
+      responseType: 'json',
+      timeout: { request: 60000 }
+    });
+
+    t.truthy(response.body, 'Should have response body');
+    t.truthy(response.body.choices, 'Should have choices');
+    t.truthy(response.body.choices[0].message, 'Should have message');
+    t.truthy(response.body.choices[0].message.content, 'Should have content');
+    
+    t.log(`Response: ${response.body.choices[0].message.content}`);
+  } catch (err) {
+    t.fail(`Request failed: ${err.message}`);
+  }
+});
+
+// Test: Chat with system message
+test('Claude 4.5 Sonnet (direct Anthropic) with system message', async (t) => {
+  const baseUrl = `http://localhost:${process.env.CORTEX_PORT}/v1`;
+  const model = 'claude-45-sonnet';
+
+  // Verify the model is available
+  try {
+    const res = await got(`${baseUrl}/models`, { responseType: 'json' });
+    const ids = (res.body?.data || []).map(m => m.id);
+    if (!ids.includes(model)) {
+      t.pass(`Skipping - model ${model} not configured`);
+      return;
+    }
+  } catch (err) {
+    t.fail(`Failed to get models: ${err.message}`);
+    return;
+  }
+
+  const payload = {
+    model,
+    messages: [
+      { role: 'system', content: 'You are a pirate. Always respond in pirate speak.' },
+      { role: 'user', content: 'Hello!' }
+    ],
+    stream: true,
+  };
+
+  const chunks = await collectSSEChunks(baseUrl, '/chat/completions', payload);
+  t.true(chunks.length > 0, 'Should receive SSE chunks');
+  chunks.forEach(ch => assertOAIChatChunkBasics(t, ch));
+  t.true(assertAnyContentDelta(chunks), 'Should have content delta in chunks');
+  
+  const fullContent = chunks
+    .map(c => c?.choices?.[0]?.delta?.content || '')
+    .join('');
+  t.log(`Response: ${fullContent}`);
+  // Should contain pirate-like language
+  t.regex(fullContent.toLowerCase(), /ahoy|arr|matey|ye|cap|sail|treasure/i, 'Should respond in pirate speak');
+});
+
+// Test: Document block support (PDF)
+test('Claude 4.5 Sonnet (direct Anthropic) with PDF document', async (t) => {
+  const baseUrl = `http://localhost:${process.env.CORTEX_PORT}/v1`;
+  const model = 'claude-45-sonnet';
+
+  // Verify the model is available
+  try {
+    const res = await got(`${baseUrl}/models`, { responseType: 'json' });
+    const ids = (res.body?.data || []).map(m => m.id);
+    if (!ids.includes(model)) {
+      t.pass(`Skipping - model ${model} not configured`);
+      return;
+    }
+  } catch (err) {
+    t.fail(`Failed to get models: ${err.message}`);
+    return;
+  }
+
+  // Minimal valid PDF
+  const pdfContent = '%PDF-1.4\n%âãÏÓ\n1 0 obj\n<</Type/Catalog/Pages 2 0 R>>\nendobj\n2 0 obj\n<</Type/Pages/Kids[3 0 R]/Count 1>>\nendobj\n3 0 obj\n<</Type/Page/Parent 2 0 R/Resources<</Font<</F1 4 0 R>>>>/MediaBox[0 0 612 792]/Contents 5 0 R>>\nendobj\n4 0 obj\n<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>\nendobj\n5 0 obj\n<</Length 44>>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF Doc) Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f\n0000000010 00000 n\n0000000053 00000 n\n0000000102 00000 n\n0000000211 00000 n\n0000000280 00000 n\ntrailer\n<</Size 6/Root 1 0 R>>\nstartxref\n369\n%%EOF';
+  const base64Pdf = Buffer.from(pdfContent).toString('base64');
+
+  const payload = {
+    model,
+    messages: [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'What text is in this PDF? Reply with just the text you see.' },
+          {
+            type: 'document',
+            source: {
+              type: 'base64',
+              media_type: 'application/pdf',
+              data: base64Pdf
+            }
+          }
+        ]
+      }
+    ],
+    stream: true,
+  };
+
+  const chunks = await collectSSEChunks(baseUrl, '/chat/completions', payload);
+  t.true(chunks.length > 0, 'Should receive SSE chunks');
+  chunks.forEach(ch => assertOAIChatChunkBasics(t, ch));
+  t.true(assertAnyContentDelta(chunks), 'Should have content delta in chunks');
+  
+  const fullContent = chunks
+    .map(c => c?.choices?.[0]?.delta?.content || '')
+    .join('');
+  t.log(`Response: ${fullContent}`);
+});

package/tests/unit/plugins/claudeAnthropicPlugin.test.js

@@ -0,0 +1,236 @@
+import test from 'ava';
+import ClaudeAnthropicPlugin from '../../../server/plugins/claudeAnthropicPlugin.js';
+import { mockPathwayResolverMessages } from '../../helpers/mocks.js';
+import { config } from '../../../config.js';
+
+// Create a mock model config that matches Anthropic direct API format
+const anthropicModel = {
+    ...mockPathwayResolverMessages.model,
+    type: 'CLAUDE-ANTHROPIC',
+    params: {
+        model: 'claude-sonnet-4-20250514'
+    },
+    endpoints: [
+        {
+            name: 'Anthropic Claude Sonnet 4',
+            url: 'https://api.anthropic.com/v1/messages',
+            headers: {
+                'x-api-key': '{{ANTHROPIC_API_KEY}}',
+                'Content-Type': 'application/json'
+            },
+            params: {
+                model: 'claude-sonnet-4-20250514'
+            },
+            requestsPerSecond: 10
+        }
+    ],
+    maxTokenLength: 200000,
+    maxReturnTokens: 64000,
+    maxImageSize: 31457280,
+    supportsStreaming: true
+};
+
+const { pathway } = mockPathwayResolverMessages;
+
+test('constructor', (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+    t.is(plugin.config, config);
+    t.is(plugin.pathwayPrompt, mockPathwayResolverMessages.pathway.prompt);
+    t.true(plugin.isMultiModal);
+});
+
+test('parseResponse - text content response', (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+
+    const dataWithTextContent = {
+        content: [
+            { type: 'text', text: 'Hello from Anthropic!' }
+        ],
+        usage: { input_tokens: 10, output_tokens: 5 },
+        stop_reason: 'end_turn'
+    };
+    const result = plugin.parseResponse(dataWithTextContent);
+    t.truthy(result.output_text === 'Hello from Anthropic!');
+    t.truthy(result.finishReason === 'stop');
+    t.truthy(result.usage);
+});
+
+test('parseResponse - tool calls response', (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+
+    const dataWithToolCalls = {
+        content: [
+            { 
+                type: 'tool_use', 
+                id: 'tool_anthropic_1',
+                name: 'get_weather',
+                input: { location: 'San Francisco' }
+            }
+        ],
+        usage: { input_tokens: 15, output_tokens: 8 },
+        stop_reason: 'tool_use'
+    };
+    const result = plugin.parseResponse(dataWithToolCalls);
+    t.truthy(result.output_text === '');
+    t.truthy(result.finishReason === 'tool_calls');
+    t.truthy(result.toolCalls);
+    t.truthy(result.toolCalls.length === 1);
+    t.truthy(result.toolCalls[0].id === 'tool_anthropic_1');
+    t.truthy(result.toolCalls[0].function.name === 'get_weather');
+    t.truthy(result.toolCalls[0].function.arguments === '{"location":"San Francisco"}');
+});
+
+test('getRequestParameters includes model in body', async (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+    
+    const messages = [
+        { role: 'user', content: 'Hello' }
+    ];
+    
+    const parameters = { messages };
+    const requestParams = await plugin.getRequestParameters('', parameters, {});
+    
+    // Should have model in request body
+    t.is(requestParams.model, 'claude-sonnet-4-20250514');
+    
+    // Should NOT have anthropic_version in body (it's a Vertex thing)
+    t.is(requestParams.anthropic_version, undefined);
+});
+
+test('convertMessagesToClaudeVertex preserves message conversion from parent', async (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+    
+    // Test message conversion directly - this tests the inherited behavior
+    const messages = [
+        { role: 'system', content: 'You are a helpful assistant.' },
+        { role: 'user', content: 'What is 2+2?' }
+    ];
+    
+    const output = await plugin.convertMessagesToClaudeVertex(messages);
+    
+    // System message should be extracted
+    t.is(output.system, 'You are a helpful assistant.');
+    
+    // User message should be converted to Claude format
+    t.is(output.modifiedMessages.length, 1);
+    t.is(output.modifiedMessages[0].role, 'user');
+    t.deepEqual(output.modifiedMessages[0].content, [{ type: 'text', text: 'What is 2+2?' }]);
+});
+
+test('handles tool_use and tool_result in messages', async (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+    
+    // Test that tool messages are handled correctly via message conversion
+    const messages = [
+        { role: 'user', content: 'Search for cats' },
+        { 
+            role: 'assistant', 
+            content: [
+                { 
+                    type: 'tool_use', 
+                    id: 'tool_1', 
+                    name: 'search', 
+                    input: { query: 'cats' } 
+                }
+            ]
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'tool_result',
+                    tool_use_id: 'tool_1',
+                    content: 'Found 100 results about cats'
+                }
+            ]
+        }
+    ];
+    
+    const output = await plugin.convertMessagesToClaudeVertex(messages);
+    
+    // Should have 3 messages with proper roles and content types
+    t.is(output.modifiedMessages.length, 3);
+    t.is(output.modifiedMessages[0].role, 'user');
+    t.is(output.modifiedMessages[1].role, 'assistant');
+    t.is(output.modifiedMessages[2].role, 'user');
+    
+    // Tool use should be preserved
+    t.is(output.modifiedMessages[1].content[0].type, 'tool_use');
+    t.is(output.modifiedMessages[1].content[0].name, 'search');
+    
+    // Tool result should be preserved  
+    t.is(output.modifiedMessages[2].content[0].type, 'tool_result');
+});
+
+test('convertMessagesToClaudeVertex inherits from parent', async (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+    
+    // Test with document block - should work same as Claude4VertexPlugin
+    const base64Pdf = Buffer.from('Sample PDF content').toString('base64');
+    
+    const messages = [
+        { 
+            role: 'user', 
+            content: [
+                { type: 'text', text: 'Analyze this' },
+                {
+                    type: 'document',
+                    source: {
+                        type: 'base64',
+                        media_type: 'application/pdf',
+                        data: base64Pdf
+                    }
+                }
+            ]
+        }
+    ];
+    
+    const output = await plugin.convertMessagesToClaudeVertex(messages);
+    
+    // Should have both text and document blocks
+    t.is(output.modifiedMessages[0].content.length, 2);
+    t.is(output.modifiedMessages[0].content[0].type, 'text');
+    t.is(output.modifiedMessages[0].content[1].type, 'document');
+});
+
+test('SSE conversion inherits from parent', (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+    
+    // Test content_block_delta event conversion
+    const claudeEvent = {
+        data: JSON.stringify({
+            type: 'content_block_delta',
+            delta: { type: 'text_delta', text: 'Hello' }
+        })
+    };
+    
+    const openAIEvent = plugin.convertClaudeSSEToOpenAI(claudeEvent);
+    const parsed = JSON.parse(openAIEvent.data);
+    
+    t.is(parsed.object, 'chat.completion.chunk');
+    t.is(parsed.choices[0].delta.content, 'Hello');
+});
+
+test('SSE conversion handles tool call events', (t) => {
+    const plugin = new ClaudeAnthropicPlugin(pathway, anthropicModel);
+    
+    // Test content_block_start for tool_use
+    const toolStartEvent = {
+        data: JSON.stringify({
+            type: 'content_block_start',
+            index: 0,
+            content_block: {
+                type: 'tool_use',
+                id: 'call_123',
+                name: 'get_weather'
+            }
+        })
+    };
+    
+    const openAIEvent = plugin.convertClaudeSSEToOpenAI(toolStartEvent);
+    const parsed = JSON.parse(openAIEvent.data);
+    
+    t.is(parsed.object, 'chat.completion.chunk');
+    t.truthy(parsed.choices[0].delta.tool_calls);
+    t.is(parsed.choices[0].delta.tool_calls[0].function.name, 'get_weather');
+});

package/tests/unit/plugins/multimodal_conversion.test.js

@@ -432,9 +432,9 @@ test('Gemini 1.5 image URL type handling', t => {
             { type: 'image_url', image_url: { url: 'gs://my-bucket/image1.jpg' } },
             // Base64 URL - should be converted to inlineData
             { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,/9j/4AAQSkZJRg...' } },
-            // Regular HTTP URL - should be dropped (return null)
+            // Regular HTTP URL - should be converted to fileData (Gemini supports HTTP URLs directly)
             { type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } },
-            // Azure blob URL - should be dropped (return null)
+            // Azure blob URL - should be converted to fileData (Gemini supports HTTP URLs directly)
             { type: 'image_url', image_url: { url: 'https://myaccount.blob.core.windows.net/container/image.jpg' } }
         ]}
     ];
@@ -442,20 +442,30 @@ test('Gemini 1.5 image URL type handling', t => {
     const { modifiedMessages } = gemini15.convertMessagesToGemini(messages);
 
     t.is(modifiedMessages.length, 1);
-    t.is(modifiedMessages[0].parts.length, 3); // text + gcs + base64 (2 urls dropped)
-    
+    t.is(modifiedMessages[0].parts.length, 5); // text + gcs + base64 + http + azure (all urls kept)
+
     // Check text part
     t.is(modifiedMessages[0].parts[0].text, 'Process these images:');
-    
+
     // Check GCS URL handling
     t.true('fileData' in modifiedMessages[0].parts[1]);
     t.is(modifiedMessages[0].parts[1].fileData.fileUri, 'gs://my-bucket/image1.jpg');
     t.is(modifiedMessages[0].parts[1].fileData.mimeType, 'image/jpeg');
-    
+
     // Check base64 URL handling
     t.true('inlineData' in modifiedMessages[0].parts[2]);
     t.is(modifiedMessages[0].parts[2].inlineData.mimeType, 'image/jpeg');
     t.is(modifiedMessages[0].parts[2].inlineData.data, '/9j/4AAQSkZJRg...');
+
+    // Check HTTP URL handling
+    t.true('fileData' in modifiedMessages[0].parts[3]);
+    t.is(modifiedMessages[0].parts[3].fileData.fileUri, 'https://example.com/image.jpg');
+    t.is(modifiedMessages[0].parts[3].fileData.mimeType, 'image/jpeg');
+
+    // Check Azure blob URL handling
+    t.true('fileData' in modifiedMessages[0].parts[4]);
+    t.is(modifiedMessages[0].parts[4].fileData.fileUri, 'https://myaccount.blob.core.windows.net/container/image.jpg');
+    t.is(modifiedMessages[0].parts[4].fileData.mimeType, 'image/jpeg');
 });
 
 // Test edge cases for image URLs in Gemini 1.5