modelmix 3.6.0 → 3.6.4

package/index.js

@@ -1,6 +1,6 @@
 const axios = require('axios');
 const fs = require('fs');
-const { fileTypeFromBuffer } = require('file-type');
+const { fromBuffer } = require('file-type');
 const log = require('lemonlog')('ModelMix');
 const Bottleneck = require('bottleneck');
 const path = require('path');
@@ -31,7 +31,6 @@ class ModelMix {
 
         this.config = {
             system: 'You are an assistant.',
-            systemExtra: '',
             max_history: 1, // Default max history
             debug: false,
             bottleneck: defaultBottleneckConfig,
@@ -203,104 +202,133 @@ class ModelMix {
     }
 
     addImageFromBuffer(buffer, { role = "user" } = {}) {
-
-        const fileType = fileTypeFromBuffer(buffer);
-        if (!fileType || !fileType.mime.startsWith('image/')) {
-            throw new Error('Invalid image buffer - unable to detect valid image format');
-        }
-
-        const data = buffer.toString('base64');
-
-        const imageMessage = {
-            ...{ role },
-            content: [
-                {
-                    type: "image",
-                    "source": {
-                        type: "base64",
-                        media_type: fileType.mime,
-                        data
-                    }
+        this.messages.push({
+            role,
+            content: [{
+                type: "image",
+                source: {
+                    type: "buffer",
+                    data: buffer
                 }
-            ]
-        };
-
-        this.messages.push(imageMessage);
+            }]
+        });
         return this;
     }
 
     addImage(filePath, { role = "user" } = {}) {
-        const imageBuffer = this.readFile(filePath, { encoding: null });
-        return this.addImageFromBuffer(imageBuffer, { role });
+        this.messages.push({
+            role,
+            content: [{
+                type: "image",
+                source: {
+                    type: "file",
+                    data: filePath
+                }
+            }]
+        });
+        return this;
     }
 
-    addImageFromUrl(url, config = { role: "user" }) {
-        if (!this.imagesToProcess) {
-            this.imagesToProcess = [];
-        }
-        this.imagesToProcess.push({ url, config });
+    addImageFromUrl(url, { role = "user" } = {}) {
+        this.messages.push({
+            role,
+            content: [{
+                type: "image",
+                source: {
+                    type: "url",
+                    data: url
+                }
+            }]
+        });
         return this;
     }
 
-    async processImageUrls() {
-        if (!this.imagesToProcess) return;
-
-        const imageContents = await Promise.all(
-            this.imagesToProcess.map(async (image) => {
+    async processImages() {
+        // Process images that are in messages
+        for (let i = 0; i < this.messages.length; i++) {
+            const message = this.messages[i];
+            if (!message.content) continue;
+            
+            for (let j = 0; j < message.content.length; j++) {
+                const content = message.content[j];
+                if (content.type !== 'image' || content.source.type === 'base64') continue;
+                
                 try {
-                    const response = await axios.get(image.url, { responseType: 'arraybuffer' });
-                    const base64 = Buffer.from(response.data, 'binary').toString('base64');
-                    const mimeType = response.headers['content-type'];
-                    return { base64, mimeType, config: image.config };
+                    let buffer, mimeType;
+                    
+                    switch (content.source.type) {
+                        case 'url':
+                            const response = await axios.get(content.source.data, { responseType: 'arraybuffer' });
+                            buffer = Buffer.from(response.data);
+                            mimeType = response.headers['content-type'];
+                            break;
+                            
+                        case 'file':
+                            buffer = this.readFile(content.source.data, { encoding: null });
+                            break;
+                            
+                        case 'buffer':
+                            buffer = content.source.data;
+                            break;
+                    }
+                    
+                    // Detect mimeType if not provided
+                    if (!mimeType) {
+                        const fileType = await fromBuffer(buffer);
+                        if (!fileType || !fileType.mime.startsWith('image/')) {
+                            throw new Error(`Invalid image - unable to detect valid image format`);
+                        }
+                        mimeType = fileType.mime;
+                    }
+                    
+                    // Update the content with processed image
+                    message.content[j] = {
+                        type: "image",
+                        source: {
+                            type: "base64",
+                            media_type: mimeType,
+                            data: buffer.toString('base64')
+                        }
+                    };
+                    
                 } catch (error) {
-                    console.error(`Error downloading image from ${image.url}:`, error);
-                    return null;
+                    console.error(`Error processing image:`, error);
+                    // Remove failed image from content
+                    message.content.splice(j, 1);
+                    j--;
                 }
-            })
-        );
-
-        imageContents.forEach((image) => {
-            if (image) {
-                const imageMessage = {
-                    ...image.config,
-                    content: [
-                        {
-                            type: "image",
-                            "source": {
-                                type: "base64",
-                                media_type: image.mimeType,
-                                data: image.base64
-                            }
-                        }
-                    ]
-                };
-                this.messages.push(imageMessage);
             }
-        });
+        }
     }
 
     async message() {
-        this.options.stream = false;
-        let raw = await this.execute();
+        let raw = await this.execute({ options: { stream: false } });
         return raw.message;
     }
 
     async json(schemaExample = null, schemaDescription = {}, { type = 'json_object', addExample = false, addSchema = true } = {}) {
-        this.options.response_format = { type };
+
+        let options = {
+            response_format: { type },
+            stream: false,
+        }
+
+        let config = {
+            system: this.config.system,
+        }
 
         if (schemaExample) {
-            this.config.schema = generateJsonSchema(schemaExample, schemaDescription);
+            config.schema = generateJsonSchema(schemaExample, schemaDescription);
 
             if (addSchema) {
-                this.config.systemExtra = "\nOutput JSON Schema: \n```\n" + JSON.stringify(this.config.schema) + "\n```";
+                config.system += "\nOutput JSON Schema: \n```\n" + JSON.stringify(this.config.schema) + "\n```";
             }
             if (addExample) {
-                this.config.systemExtra += "\nOutput JSON Example: \n```\n" + JSON.stringify(schemaExample) + "\n```";
+                config.system += "\nOutput JSON Example: \n```\n" + JSON.stringify(schemaExample) + "\n```";
             }
         }
-        const response = await this.message();
-        this.config.systemExtra = "";
-        return JSON.parse(this._extractBlock(response));
+        const { message } = await this.execute({ options, config });
+        return JSON.parse(this._extractBlock(message));
     }
 
     _extractBlock(response) {
@@ -309,23 +337,24 @@ class ModelMix {
     }
 
     async block({ addSystemExtra = true } = {}) {
+        let config = {
+            system: this.config.system,
+        }
+
         if (addSystemExtra) {
-            this.config.systemExtra = "\nReturn the result of the task between triple backtick block code tags ```";
+            config.system += "\nReturn the result of the task between triple backtick block code tags ```";
         }
-        const response = await this.message();
-        this.config.systemExtra = "";
-        return this._extractBlock(response);
+        const { message } = await this.execute({ options: { stream: false }, config });
+        return this._extractBlock(message);
     }
 
     async raw() {
-        this.options.stream = false;
-        return this.execute();
+        return this.execute({ options: { stream: false } });
     }
 
     async stream(callback) {
-        this.options.stream = true;
         this.streamCallback = callback;
-        return this.execute();
+        return this.execute({ options: { stream: true } });
     }
 
     replaceKeyFromFile(key, filePath) {
@@ -376,7 +405,7 @@ class ModelMix {
     }
 
     async prepareMessages() {
-        await this.processImageUrls();
+        await this.processImages();
         this.applyTemplate();
         this.messages = this.messages.slice(-this.config.max_history);
         this.messages = this.groupByRoles(this.messages);
@@ -398,9 +427,9 @@ class ModelMix {
         }
     }
 
-    async execute() {
+    async execute({ config = {}, options = {} } = {}) {
         if (!this.models || this.models.length === 0) {
-            throw new Error("No models specified. Use methods like .gpt(), .sonnet() first.");
+            throw new Error("No models specified. Use methods like .gpt41mini(), .sonnet4() first.");
         }
 
         return this.limiter.schedule(async () => {
@@ -419,16 +448,18 @@ class ModelMix {
                 const providerInstance = currentModel.provider;
                 const optionsTools = providerInstance.getOptionsTools(this.tools);
 
-                let options = {
+                options = {
                     ...this.options,
                     ...providerInstance.options,
                     ...optionsTools,
+                    ...options,
                     model: currentModelKey
                 };
 
-                const config = {
+                config = {
                     ...this.config,
                     ...providerInstance.config,
+                    ...config,
                 };
 
                 if (config.debug) {
@@ -761,7 +792,7 @@ class MixOpenAI extends MixCustom {
 
     static convertMessages(messages, config) {
 
-        const content = config.system + config.systemExtra;
+        const content = config.system;
         messages = [{ role: 'system', content }, ...messages || []];
 
         const results = []
@@ -851,7 +882,7 @@ class MixAnthropic extends MixCustom {
 
         delete options.response_format;
 
-        options.system = config.system + config.systemExtra;
+        options.system = config.system;
         return super.create({ config, options });
     }
 
@@ -982,16 +1013,12 @@ class MixPerplexity extends MixCustom {
     async create({ config = {}, options = {} } = {}) {
 
         if (config.schema) {
-            config.systemExtra = '';
-
             options.response_format = {
                 type: 'json_schema',
                 json_schema: { schema: config.schema }
             };
         }
 
-        const content = config.system + config.systemExtra;
-        options.messages = [{ role: 'system', content }, ...options.messages || []];
         return super.create({ config, options });
     }
 }
@@ -1025,7 +1052,7 @@ class MixOllama extends MixCustom {
     }
 
     static convertMessages(messages, config) {
-        const content = config.system + config.systemExtra;
+        const content = config.system;
         messages = [{ role: 'system', content }, ...messages || []];
 
         return messages.map(entry => {
@@ -1216,7 +1243,7 @@ class MixGoogle extends MixCustom {
         const fullUrl = `${this.config.url}/${options.model}:${generateContentApi}?key=${this.config.apiKey}`;
 
 
-        const content = config.system + config.systemExtra;
+        const content = config.system;
         const systemInstruction = { parts: [{ text: content }] };
 
         options.messages = MixGoogle.convertMessages(options.messages);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelmix",
-  "version": "3.6.0",
+  "version": "3.6.4",
   "description": "🧬 ModelMix - Unified API for Diverse AI LLM.",
   "main": "index.js",
   "repository": {