modelmix 3.8.2 → 3.8.4

package/test/fallback.test.js

@@ -0,0 +1,387 @@
+const { expect } = require('chai');
+const sinon = require('sinon');
+const nock = require('nock');
+const { ModelMix } = require('../index.js');
+
+describe('Provider Fallback Chain Tests', () => {
+
+    afterEach(() => {
+        nock.cleanAll();
+        sinon.restore();
+    });
+
+    describe('Basic Fallback Chain', () => {
+        let model;
+
+        beforeEach(() => {
+            model = ModelMix.new({
+                config: { debug: false }
+            });
+        });
+
+        it('should use primary provider when available', async () => {
+            model.gpt5mini().sonnet4().addText('Hello');
+
+            // Mock successful OpenAI response
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(200, {
+                    choices: [{
+                        message: {
+                            role: 'assistant',
+                            content: 'Hello from GPT-5 mini!'
+                        }
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Hello from GPT-5 mini!');
+        });
+
+        it('should fallback to secondary provider when primary fails', async () => {
+            model.gpt5mini().sonnet4().addText('Hello');
+
+            // Mock failed OpenAI response (GPT-5 mini)
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(500, { error: 'Server error' });
+
+            // Mock successful Anthropic response (Sonnet 4)
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(200, {
+                    content: [{
+                        type: 'text',
+                        text: 'Hello from Claude Sonnet 4!'
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Hello from Claude Sonnet 4!');
+        });
+
+        it('should cascade through multiple fallbacks', async () => {
+            model.gpt5mini().sonnet4().gemini25flash().addText('Hello');
+
+            // Mock failed OpenAI response
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(429, { error: 'Rate limit exceeded' });
+
+            // Mock failed Anthropic response
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(401, { error: 'Unauthorized' });
+
+            // Mock successful Google response
+            nock('https://generativelanguage.googleapis.com')
+                .post(/.*generateContent/)
+                .reply(200, {
+                    candidates: [{
+                        content: {
+                            parts: [{
+                                text: 'Hello from Google Gemini 2.5 Flash!'
+                            }]
+                        }
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Hello from Google Gemini 2.5 Flash!');
+        });
+
+        it('should throw error when all providers fail', async () => {
+            model.gpt5mini().sonnet4().addText('Hello');
+
+            // Mock all providers failing
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(500, { error: 'All servers down' });
+
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(500, { error: 'All servers down' });
+
+            try {
+                await model.message();
+                expect.fail('Should have thrown an error');
+            } catch (error) {
+                expect(error.message).to.include('500');
+            }
+        });
+    });
+
+    describe('Cross-Provider Fallback', () => {
+        let model;
+
+        beforeEach(() => {
+            model = ModelMix.new({
+                config: { debug: false }
+            });
+        });
+
+        it('should fallback from OpenAI to Anthropic', async () => {
+            model.gpt5mini().sonnet4().addText('Test message');
+
+            // Mock OpenAI failure
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(503, { error: 'Service unavailable' });
+
+            // Mock Anthropic success
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(200, {
+                    content: [{
+                        type: 'text',
+                        text: 'Response from Anthropic'
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Response from Anthropic');
+        });
+
+        it('should fallback from Anthropic to Google', async () => {
+            model.sonnet4().gemini25flash().addText('Test message');
+
+            // Mock Anthropic failure
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(401, { error: 'Unauthorized' });
+
+            // Mock Google success
+            nock('https://generativelanguage.googleapis.com')
+                .post(/.*generateContent/)
+                .reply(200, {
+                    candidates: [{
+                        content: {
+                            parts: [{
+                                text: 'Response from Google Gemini 2.5 Flash'
+                            }]
+                        }
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Response from Google Gemini 2.5 Flash');
+        });
+
+        it('should handle network timeout fallback', async () => {
+            model.gpt5mini().sonnet4().addText('Hello');
+
+            // Mock timeout error on first provider (using 408 Request Timeout)
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(408, { error: 'Request timeout' });
+
+            // Mock successful response on fallback (Anthropic)
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(200, {
+                    content: [{
+                        type: 'text',
+                        text: 'Quick fallback response from Claude'
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Quick fallback response from Claude');
+        });
+    });
+
+    describe('Fallback with Different Response Formats', () => {
+        let model;
+
+        beforeEach(() => {
+            model = ModelMix.new({
+                config: { debug: false }
+            });
+        });
+
+        it('should handle JSON fallback correctly', async () => {
+            const schema = { name: 'Alice', age: 30 };
+            model.gpt5mini().sonnet4().addText('Generate user data');
+
+            // Mock OpenAI failure
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(400, { error: 'Bad request' });
+
+            // Mock Anthropic success with JSON
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(200, {
+                    content: [{
+                        type: 'text',
+                        text: JSON.stringify({ name: 'Bob', age: 25 })
+                    }]
+                });
+
+            const result = await model.json(schema);
+
+            expect(result).to.have.property('name');
+            expect(result).to.have.property('age');
+            expect(result.name).to.equal('Bob');
+            expect(result.age).to.equal(25);
+        });
+
+        it('should preserve message history through fallbacks', async () => {
+            model.gpt5mini().sonnet4()
+                .addText('First message')
+                .addText('Second message');
+
+            // Mock first provider failure
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(500, { error: 'Server error' });
+
+            // Mock second provider success (Anthropic)
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(200, {
+                    content: [{
+                        type: 'text',
+                        text: 'Fallback response with context from Claude'
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Fallback response with context from Claude');
+        });
+    });
+
+    describe('Fallback Configuration', () => {
+        it('should respect custom provider configurations in fallback', async () => {
+            const model = ModelMix.new({
+                config: { debug: false },
+                options: { temperature: 0.5 }
+            });
+
+            // Configure with custom temperature for fallback
+            model.gpt5mini({ options: { temperature: 0.6 } })
+                 .sonnet4({ options: { temperature: 0.7 } })
+                 .addText('Creative response');
+
+            // Mock first provider failure
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(503, { error: 'Service unavailable' });
+
+            // Mock second provider success - verify temperature in request (Anthropic)
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(function(uri, requestBody) {
+                    const body = typeof requestBody === 'string' ? JSON.parse(requestBody) : requestBody;
+                    expect(body.temperature).to.equal(0.7);
+                    return [200, {
+                        content: [{
+                            type: 'text',
+                            text: 'Creative fallback response from Claude'
+                        }]
+                    }];
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Creative fallback response from Claude');
+        });
+
+        it('should handle provider-specific options in fallback chain', async () => {
+            const model = ModelMix.new({
+                config: { debug: false }
+            });
+
+            model.gpt5mini({ options: { max_tokens: 100 } })
+                .sonnet4({ options: { max_tokens: 200 } })
+                .addText('Generate text');
+
+            // Mock OpenAI failure
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(429, { error: 'Rate limited' });
+
+            // Mock Anthropic success - verify max_tokens
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(function (uri, body) {
+                    expect(body.max_tokens).to.equal(200);
+                    return [200, {
+                        content: [{
+                            type: 'text',
+                            text: 'Fallback with correct max_tokens'
+                        }]
+                    }];
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Fallback with correct max_tokens');
+        });
+    });
+
+    describe('Error Handling in Fallback', () => {
+        let model;
+
+        beforeEach(() => {
+            model = ModelMix.new({
+                config: { debug: false }
+            });
+        });
+
+        it('should provide detailed error information when all fallbacks fail', async () => {
+            model.gpt5mini().sonnet4().gemini25flash().addText('Test');
+
+            // Mock all providers failing with different errors
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(500, { error: 'OpenAI server error' });
+
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(401, { error: 'Anthropic auth error' });
+
+            nock('https://generativelanguage.googleapis.com')
+                .post(/.*generateContent/)
+                .reply(403, { error: 'Google forbidden' });
+
+            try {
+                await model.message();
+                expect.fail('Should have thrown an error');
+            } catch (error) {
+                // Should contain information about the final error
+                expect(error.message).to.be.a('string');
+            }
+        });
+
+        it('should handle malformed responses in fallback', async () => {
+            model.gpt5mini().sonnet4().addText('Test');
+
+            // Mock malformed response from first provider
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(200, { invalid: 'response' });
+
+            // Mock valid response from fallback (Anthropic)
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(200, {
+                    content: [{
+                        type: 'text',
+                        text: 'Valid fallback response from Claude'
+                    }]
+                });
+
+            const response = await model.message();
+
+            expect(response).to.include('Valid fallback response from Claude');
+        });
+    });
+});

package/test/fixtures/data.json

@@ -0,0 +1,36 @@
+{
+  "users": [
+    {
+      "id": 1,
+      "name": "Alice Smith",
+      "email": "alice@example.com",
+      "role": "admin",
+      "active": true
+    },
+    {
+      "id": 2,
+      "name": "Bob Johnson",
+      "email": "bob@example.com",
+      "role": "user",
+      "active": false
+    },
+    {
+      "id": 3,
+      "name": "Carol Davis",
+      "email": "carol@example.com",
+      "role": "moderator",
+      "active": true
+    }
+  ],
+  "settings": {
+    "theme": "dark",
+    "notifications": true,
+    "language": "en",
+    "timezone": "UTC"
+  },
+  "metadata": {
+    "version": "1.0.0",
+    "last_updated": "2023-12-01T10:00:00Z",
+    "total_users": 3
+  }
+}

package/test/fixtures/template.txt

@@ -0,0 +1,15 @@
+Hello {{name}}, welcome to {{platform}}!
+
+Your account details:
+- Username: {{username}}
+- Role: {{role}}
+- Created: {{created_date}}
+
+Please visit {{website}} for more information.
+
+{{#if premium}}
+Thank you for being a premium member!
+{{/if}}
+
+Best regards,
+The {{company}} Team

package/test/images.test.js

@@ -0,0 +1,87 @@
+const { expect } = require('chai');
+const sinon = require('sinon');
+const nock = require('nock');
+const { ModelMix } = require('../index.js');
+
+describe('Image Processing and Multimodal Support Tests', () => {
+
+    afterEach(() => {
+        nock.cleanAll();
+        sinon.restore();
+    });
+
+    describe('Image Data Handling', () => {
+        let model;
+
+        const max_history = 2;
+
+        beforeEach(() => {
+            model = ModelMix.new({
+                config: { debug: false },
+                config: { max_history }
+            });
+        });
+
+        it('should handle base64 image data correctly', async () => {
+            const base64Image = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC';
+
+            model.gpt4o()
+                .addText('What do you see in this image?')
+                .addImageFromUrl(base64Image);
+
+            nock('https://api.openai.com')
+                .post('/v1/chat/completions')
+                .reply(function (uri, body) {
+                    expect(body.messages[1].content).to.be.an('array');
+                    expect(body.messages[1].content).to.have.length(max_history);
+                    expect(body.messages[1].content[max_history - 1].image_url.url).to.equal(base64Image);
+
+                    return [200, {
+                        choices: [{
+                            message: {
+                                role: 'assistant',
+                                content: 'I can see a small test image'
+                            }
+                        }]
+                    }];
+                });
+
+            const response = await model.message();
+            expect(response).to.include('I can see a small test image');
+        });
+
+        it('should support multimodal with sonnet4()', async () => {
+            const base64Image = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC';
+
+            model.sonnet4()
+                .addText('Describe this image')
+                .addImageFromUrl(base64Image);
+
+            // Claude expects images as base64 in the content array
+            // We'll check that the message is formatted as expected
+            nock('https://api.anthropic.com')
+                .post('/v1/messages')
+                .reply(function (uri, body) {
+                    console.log(body.messages);
+                    // body is already parsed as JSON by nock
+                    expect(body.messages).to.be.an('array');
+                    // Find the message with the image
+                    const userMsg = body.messages.find(m => m.role === 'user');
+                    expect(userMsg).to.exist;
+                    const imageContent = userMsg.content.find(c => c.type === 'image');
+                    expect(imageContent).to.exist;
+                    expect(imageContent.source.type).to.equal('base64');
+                    expect(imageContent.source.data).to.equal(base64Image.split(',')[1]);
+                    expect(imageContent.source.media_type).to.equal('image/png');
+                    return [200, {
+                        content: [{ type: "text", text: "This is a small PNG test image." }],
+                        role: "assistant"
+                    }];
+                });
+
+            const response = await model.message();
+            expect(response).to.include('small PNG test image');
+        });
+
+    });
+});