@pheem49/mint 1.5.1 → 1.5.3

package/src/AI_Brain/autonomous_brain.js

@@ -1,17 +1,16 @@
-const { GoogleGenAI } = require('@google/genai');
 const { readConfig } = require('../System/config_manager');
 const { performWebAutomation } = require('../Automation_Layer/browser_automation');
 const { createFolder, deleteFile } = require('../Automation_Layer/file_operations');
 const { searchKnowledge } = require('./knowledge_base');
 const safetyManager = require('../System/safety_manager');
+const providerAdapter = require('./provider_adapter');
+const taskManager = require('../System/task_manager');
 const fs = require('fs');
 const path = require('path');
 
 const os = require('os');
 const { sendNotification } = require('../System/notifications');
 
-const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
-
 function expandHome(filePath) {
     if (filePath.startsWith('~/')) {
         return path.join(os.homedir(), filePath.slice(2));
@@ -47,24 +46,23 @@ TOOL DETAILS:
 - "done": Target is the final summary of what was accomplished.
 `;
 
-async function executeAutonomousTask(taskDescription, notifyCallback) {
+async function executeAutonomousTask(taskDescription, notifyCallback, options = {}) {
     const config = readConfig();
-    const modelName = config.geminiModel || DEFAULT_GEMINI_MODEL;
-    const apiKey = config.apiKey || process.env.GEMINI_API_KEY;
-    
-    // Use the custom chat creation pattern from the project
-    const ai = new GoogleGenAI({ apiKey });
-    const chat = ai.chats.create({
-        model: modelName,
-        config: {
-            systemInstruction: AUTONOMOUS_SYSTEM_PROMPT,
-            responseMimeType: "application/json"
-        },
-        history: []
+    const providerOrder = providerAdapter.getProviderAttemptOrder(config, {
+        supported: ['gemini', 'anthropic', 'openai', 'local_openai'],
+        priority: ['anthropic', 'openai', 'gemini', 'local_openai']
+    });
+    const client = new providerAdapter.AgentProviderClient({
+        provider: providerOrder[0],
+        providerOrder,
+        config,
+        systemInstruction: AUTONOMOUS_SYSTEM_PROMPT,
+        responseMimeType: 'application/json',
+        maxTokens: 4096
     });
 
     let currentObservation = `Task: ${taskDescription}\nWhat is your first step?`;
-    let maxSteps = 10;
+    let maxSteps = Number.isFinite(options.maxSteps) ? options.maxSteps : 10;
     let step = 0;
     let result = null;
 
@@ -73,12 +71,20 @@ async function executeAutonomousTask(taskDescription, notifyCallback) {
         if (notifyCallback) notifyCallback(`Step ${step}: Thinking...`);
 
         try {
-            const response = await chat.sendMessage({ message: [{ text: currentObservation }] });
-            const text = response.text;
+            const text = await client.sendMessage(currentObservation);
             const actionObj = JSON.parse(text);
 
             console.log(`[Brain] Thought: ${actionObj.thought}`);
             console.log(`[Brain] Action: ${actionObj.action} -> ${actionObj.target}`);
+            if (options.taskId) {
+                taskManager.addCheckpoint(options.taskId, {
+                    phase: 'autonomous_step',
+                    step,
+                    thought: actionObj.thought,
+                    action: actionObj.action,
+                    target: actionObj.target
+                });
+            }
 
             if (actionObj.action === 'done') {
                 result = actionObj.target;
@@ -111,6 +117,13 @@ async function executeAutonomousTask(taskDescription, notifyCallback) {
                             approved: true
                         });
                         fs.writeFileSync(filePath, actionObj.data || '');
+                        if (options.taskId) {
+                            taskManager.addArtifact(options.taskId, {
+                                type: 'file',
+                                path: filePath,
+                                description: `Written by autonomous task step ${step}`
+                            });
+                        }
                         observation = `File written successfully to ${actionObj.target}`;
                     } catch (e) {
                         observation = `Failed to write file: ${e.message}`;
@@ -139,10 +152,24 @@ async function executeAutonomousTask(taskDescription, notifyCallback) {
             }
 
             currentObservation = `Observation: ${observation}`;
+            if (options.taskId) {
+                taskManager.addCheckpoint(options.taskId, {
+                    phase: 'observation',
+                    step,
+                    message: observation
+                });
+            }
 
         } catch (err) {
             console.error('[AutonomousBrain] Error during loop:', err);
             currentObservation = `Error occurred: ${err.message}. Please try a different approach or conclude if task is impossible.`;
+            if (options.taskId) {
+                taskManager.addCheckpoint(options.taskId, {
+                    phase: 'error',
+                    step,
+                    message: err.message
+                });
+            }
         }
     }
 

package/src/AI_Brain/headless_agent.js

@@ -23,6 +23,10 @@ let isProcessingTask = false;
 async function startAgent() {
     console.log(`\n${colors.mint}${colors.bright}[Mint-Agent] Background agent started.${colors.reset}`);
     console.log(`${colors.gray}[Mint-Agent] Monitoring system events and task queue...${colors.reset}\n`);
+    const resumed = taskManager.resumeRunningTasks();
+    if (resumed.length > 0) {
+        console.log(`${colors.gray}[Mint-Agent] Re-queued ${resumed.length} interrupted task(s).${colors.reset}`);
+    }
 
     // Initialize System Monitoring
     systemEvents.startMonitoring();
@@ -70,25 +74,40 @@ async function checkTaskQueue() {
     console.log(`\n${colors.mint}[Agent] Picking up task: ${task.description}${colors.reset}`);
     
     taskManager.updateTask(task.id, { status: 'running' });
+    taskManager.addCheckpoint(task.id, {
+        phase: 'started',
+        message: task.description
+    });
     sendNotification("🚀 เริ่มทำงานให้แล้วนะคะ", `กำลังดำเนินการ: ${task.description}`);
 
     try {
         const result = await executeAutonomousTask(task.description, (progress) => {
             console.log(`${colors.gray}[Progress] ${progress}${colors.reset}`);
+            taskManager.addCheckpoint(task.id, {
+                phase: 'progress',
+                message: progress
+            });
             // Send periodic progress notifications if important
             if (progress.includes('เสนอให้รันคำสั่ง')) {
                 sendNotification("💡 มิ้นท์มีข้อแนะนำค่ะ", progress);
             }
-        });
+        }, { taskId: task.id });
 
+        taskManager.addArtifact(task.id, {
+            type: 'final_result',
+            content: result
+        });
         taskManager.updateTask(task.id, { status: 'completed', result });
         sendNotification("✅ งานเสร็จเรียบร้อยแล้วค่ะ!", result);
         console.log(`\n${colors.mint}[Agent] Task completed successfully.${colors.reset}`);
 
     } catch (err) {
         console.error('[Agent] Task execution failed:', err);
-        taskManager.updateTask(task.id, { status: 'failed', result: err.message });
+        const next = taskManager.failTaskWithRetry(task.id, err.message);
         sendNotification("❌ เกิดข้อผิดพลาดในการทำงาน", err.message);
+        if (next && next.status === 'pending') {
+            console.log(`${colors.gray}[Agent] Task scheduled for retry (${next.retryCount}/${next.maxRetries}).${colors.reset}`);
+        }
     } finally {
         isProcessingTask = false;
     }

package/src/AI_Brain/proactive_engine.js

@@ -5,9 +5,14 @@ const { readConfig } = require('../System/config_manager');
 // Proactive Engine — Smart Suggestion Engine (Multi-Choice)
 // ============================================================
 
-const ai = new GoogleGenAI({});
 const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
 let lastLoggedModel = '';
+let _ai = null;
+
+function getAi(apiKey) {
+    if (!_ai) _ai = new GoogleGenAI({ apiKey });
+    return _ai;
+}
 
 const PROACTIVE_SYSTEM_PROMPT = `You are a Smart Suggestion Engine built into a Desktop AI Agent called "Mint".
 Your job: observe the user's screen + behavior, then offer MULTIPLE relevant quick-action options — NOT just one question.
@@ -89,11 +94,15 @@ function getMinSuggestionIntervalMs() {
  */
 async function analyzeAndSuggest(base64Image, behaviorSummary) {
     try {
-        const model = resolveGeminiModel();
+        const cfg = readConfig();
+        const apiKey = cfg.apiKey || process.env.GEMINI_API_KEY;
+        if (!apiKey) return null; // silently skip if no API key configured
+        const model = (cfg.geminiModel || '').trim() || DEFAULT_GEMINI_MODEL;
         if (model && model !== lastLoggedModel) {
             console.log(`[Gemini] Proactive Engine model: ${model}`);
             lastLoggedModel = model;
         }
+        const ai = getAi(apiKey);
 
         const now = Date.now();
         const minInterval = getMinSuggestionIntervalMs();
@@ -127,6 +136,7 @@ Rules: Only suggest if you see a clear opportunity. Return 2–4 relevant chips.
             contents: [{ role: 'user', parts: userMessage }]
         });
 
+
         let parsed;
         try {
             parsed = JSON.parse(response.text);

package/src/AI_Brain/provider_adapter.js

@@ -0,0 +1,358 @@
+const axios = require('axios');
+const { GoogleGenAI } = require('@google/genai');
+const { getAvailableProviders } = require('../System/config_manager');
+
+const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
+const ALL_PROVIDERS = ['anthropic', 'openai', 'gemini', 'local_openai', 'ollama', 'huggingface'];
+
+function splitDataUri(dataUri = '') {
+    const match = String(dataUri).match(/^data:([^;]+);base64,([\s\S]+)$/);
+    if (!match) return null;
+    return {
+        mimeType: match[1],
+        data: match[2]
+    };
+}
+
+function contentToText(content) {
+    if (content && typeof content === 'object' && !Array.isArray(content)) {
+        return String(content.text || '');
+    }
+    return String(content || '');
+}
+
+function contentToGeminiParts(content) {
+    const text = contentToText(content);
+    const parts = text ? [{ text }] : [];
+    if (content && typeof content === 'object') {
+        const images = Array.isArray(content.imageDataUris)
+            ? content.imageDataUris
+            : (content.imageDataUri ? [content.imageDataUri] : []);
+        for (const item of images) {
+            const image = splitDataUri(item);
+            if (image) parts.push({ inlineData: { mimeType: image.mimeType, data: image.data } });
+        }
+        if (content.audioDataUri) {
+            const audio = splitDataUri(content.audioDataUri);
+            if (audio) parts.push({ inlineData: { mimeType: audio.mimeType, data: audio.data } });
+        }
+    }
+    return parts.length > 0 ? parts : [{ text: '' }];
+}
+
+function contentToOpenAIContent(content) {
+    const text = contentToText(content) || 'Analyze this input.';
+    if (content && typeof content === 'object') {
+        const images = Array.isArray(content.imageDataUris)
+            ? content.imageDataUris
+            : (content.imageDataUri ? [content.imageDataUri] : []);
+        if (images.length > 0) {
+            return [
+                { type: 'text', text },
+                ...images.map(item => ({ type: 'image_url', image_url: { url: item } }))
+            ];
+        }
+    }
+    return text;
+}
+
+function contentToAnthropicContent(content) {
+    const text = contentToText(content) || 'Analyze this input.';
+    if (content && typeof content === 'object') {
+        const images = Array.isArray(content.imageDataUris)
+            ? content.imageDataUris
+            : (content.imageDataUri ? [content.imageDataUri] : []);
+        if (images.length > 0) {
+            const blocks = [];
+            for (const item of images) {
+                const image = splitDataUri(item);
+                if (image) {
+                    blocks.push({ type: 'image', source: { type: 'base64', media_type: image.mimeType, data: image.data } });
+                }
+            }
+            blocks.push({ type: 'text', text });
+            return blocks;
+        }
+    }
+    return text;
+}
+
+function contentToOllamaMessage(content) {
+    const text = contentToText(content) || 'Analyze this input.';
+    const message = { role: 'user', content: text };
+    if (content && typeof content === 'object') {
+        const images = Array.isArray(content.imageDataUris)
+            ? content.imageDataUris
+            : (content.imageDataUri ? [content.imageDataUri] : []);
+        const imagePayloads = images
+            .map(item => splitDataUri(item))
+            .filter(Boolean)
+            .map(image => image.data);
+        if (imagePayloads.length > 0) message.images = imagePayloads;
+    }
+    return message;
+}
+
+function getProviderAttemptOrder(config = {}, options = {}) {
+    const supported = options.supported || ALL_PROVIDERS;
+    const available = (options.availableProviders || getAvailableProviders(config))
+        .filter(provider => supported.includes(provider));
+    const requested = options.requested || config.aiProvider || 'gemini';
+    const priority = (options.priority || ALL_PROVIDERS).filter(provider => supported.includes(provider));
+    const ordered = [];
+
+    if (supported.includes(requested) && available.includes(requested)) {
+        ordered.push(requested);
+    }
+
+    for (const provider of priority) {
+        if (available.includes(provider) && !ordered.includes(provider)) {
+            ordered.push(provider);
+        }
+    }
+
+    return ordered.length > 0 ? ordered : ['gemini'];
+}
+
+function getProviderModel(provider, config = {}) {
+    switch (provider) {
+        case 'anthropic':
+            return config.anthropicModel || 'claude-3-5-sonnet-latest';
+        case 'openai':
+            return config.openaiModel || 'gpt-4o';
+        case 'local_openai':
+            return config.localModelName || 'local-model';
+        case 'ollama':
+            return config.ollamaModel || 'llama3:latest';
+        case 'huggingface':
+            return config.hfModel || 'meta-llama/Meta-Llama-3-8B-Instruct';
+        case 'gemini':
+        default:
+            return config.geminiModel || DEFAULT_GEMINI_MODEL;
+    }
+}
+
+class AgentProviderClient {
+    constructor(options = {}) {
+        this.provider = options.provider || 'gemini';
+        this.providerOrder = options.providerOrder && options.providerOrder.length
+            ? options.providerOrder
+            : [this.provider];
+        this.config = options.config || {};
+        this.history = options.history || [];
+        this.systemInstruction = options.systemInstruction || '';
+        this.responseMimeType = options.responseMimeType || 'application/json';
+        this.maxTokens = options.maxTokens || 8192;
+        this.lastSuccessfulProvider = null;
+        this.usageTotals = {};
+    }
+
+    recordUsage(provider, model, usage = {}) {
+        const key = `${provider}:${model || ''}`;
+        if (!this.usageTotals[key]) {
+            this.usageTotals[key] = {
+                provider,
+                model,
+                requests: 0,
+                inputTokens: 0,
+                cacheReads: 0,
+                outputTokens: 0,
+                reasoningTokens: 0,
+                totalTokens: 0
+            };
+        }
+
+        const row = this.usageTotals[key];
+        row.requests += 1;
+        row.inputTokens += Number(usage.inputTokens) || 0;
+        row.cacheReads += Number(usage.cacheReads) || 0;
+        row.outputTokens += Number(usage.outputTokens) || 0;
+        row.reasoningTokens += Number(usage.reasoningTokens) || 0;
+        row.totalTokens += Number(usage.totalTokens) || 0;
+    }
+
+    getUsageSummary() {
+        return Object.values(this.usageTotals);
+    }
+
+    async sendMessage(observation) {
+        this.history.push({ role: 'user', content: observation });
+
+        const failures = [];
+        for (const provider of this.providerOrder) {
+            this.provider = provider;
+            try {
+                let responseText = '';
+                if (provider === 'anthropic') responseText = await this.callAnthropic();
+                else if (provider === 'openai' || provider === 'local_openai') responseText = await this.callOpenAI();
+                else if (provider === 'ollama') responseText = await this.callOllama();
+                else if (provider === 'huggingface') responseText = await this.callHuggingFace();
+                else responseText = await this.callGemini();
+
+                this.history.push({ role: 'assistant', content: responseText });
+                this.lastSuccessfulProvider = provider;
+                return responseText;
+            } catch (error) {
+                const message = error.message || error.code || 'unknown error';
+                failures.push(`${provider}: ${message}`);
+                if (process.env.MINT_DEBUG === '1') {
+                    console.error(`[ProviderAdapter] Provider '${provider}' failed: ${message}`);
+                }
+            }
+        }
+
+        throw new Error(`All providers failed. ${failures.join(' | ')}`);
+    }
+
+    async callAnthropic() {
+        const apiKey = this.config.anthropicApiKey || process.env.ANTHROPIC_API_KEY;
+        const model = getProviderModel('anthropic', this.config);
+        const messages = this.history.map(m => ({
+            role: m.role,
+            content: contentToAnthropicContent(m.content)
+        }));
+
+        const response = await axios.post('https://api.anthropic.com/v1/messages', {
+            model,
+            max_tokens: this.maxTokens,
+            system: this.systemInstruction,
+            messages
+        }, {
+            headers: {
+                'x-api-key': apiKey,
+                'anthropic-version': '2023-06-01',
+                'content-type': 'application/json'
+            }
+        });
+        const usage = response.data.usage || {};
+        this.recordUsage('anthropic', model, {
+            inputTokens: usage.input_tokens,
+            cacheReads: usage.cache_read_input_tokens,
+            outputTokens: usage.output_tokens,
+            totalTokens: (Number(usage.input_tokens) || 0) + (Number(usage.output_tokens) || 0)
+        });
+        return response.data.content[0].text;
+    }
+
+    async callOpenAI() {
+        const isLocal = this.provider === 'local_openai';
+        const apiKey = isLocal ? 'not-needed' : (this.config.openaiApiKey || process.env.OPENAI_API_KEY);
+        const baseUrl = isLocal ? (this.config.localApiBaseUrl || 'http://localhost:1234/v1') : 'https://api.openai.com/v1';
+        const model = getProviderModel(this.provider, this.config);
+        const messages = [
+            { role: 'system', content: this.systemInstruction },
+            ...this.history.map(m => ({
+                role: m.role,
+                content: contentToOpenAIContent(m.content)
+            }))
+        ];
+
+        const response = await axios.post(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
+            model,
+            messages,
+            response_format: isLocal ? undefined : { type: 'json_object' }
+        }, {
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': 'application/json'
+            }
+        });
+        const usage = response.data.usage || {};
+        this.recordUsage(this.provider, model, {
+            inputTokens: usage.prompt_tokens,
+            cacheReads: usage.prompt_tokens_details && usage.prompt_tokens_details.cached_tokens,
+            outputTokens: usage.completion_tokens,
+            reasoningTokens: usage.completion_tokens_details && usage.completion_tokens_details.reasoning_tokens,
+            totalTokens: usage.total_tokens
+        });
+        return response.data.choices[0].message.content;
+    }
+
+    async callGemini() {
+        const apiKey = this.config.apiKey || process.env.GEMINI_API_KEY;
+        const model = getProviderModel('gemini', this.config);
+        const ai = new GoogleGenAI({ apiKey });
+        const recentHistory = this.history.slice(-16);
+        const priorHistory = recentHistory.slice(0, -1);
+        const lastEntry = recentHistory[recentHistory.length - 1] || { content: '' };
+        const history = priorHistory.map(m => ({
+            role: m.role === 'assistant' ? 'model' : 'user',
+            parts: contentToGeminiParts(m.content)
+        }));
+        const chat = ai.chats.create({
+            model,
+            config: {
+                systemInstruction: this.systemInstruction,
+                responseMimeType: this.responseMimeType
+            },
+            history
+        });
+
+        const response = await chat.sendMessage({ message: contentToGeminiParts(lastEntry.content) });
+        const usage = response.usageMetadata || {};
+        this.recordUsage('gemini', model, {
+            inputTokens: usage.promptTokenCount,
+            cacheReads: usage.cachedContentTokenCount,
+            outputTokens: usage.candidatesTokenCount,
+            reasoningTokens: usage.thoughtsTokenCount,
+            totalTokens: usage.totalTokenCount
+        });
+        return typeof response.text === 'function' ? response.text() : response.text;
+    }
+
+    async callOllama() {
+        const model = getProviderModel('ollama', this.config);
+        const baseUrl = (this.config.ollamaHost || 'http://localhost:11434').replace(/\/$/, '');
+        const messages = [
+            { role: 'system', content: this.systemInstruction },
+            ...this.history.map(m => m.role === 'assistant'
+                ? { role: 'assistant', content: contentToText(m.content) }
+                : contentToOllamaMessage(m.content))
+        ];
+        const response = await axios.post(`${baseUrl}/api/chat`, {
+            model,
+            messages,
+            format: this.responseMimeType === 'application/json' ? 'json' : undefined,
+            stream: false
+        });
+        return response.data.message.content;
+    }
+
+    async callHuggingFace() {
+        const apiKey = this.config.hfApiKey || process.env.HF_API_KEY;
+        const model = getProviderModel('huggingface', this.config);
+        const messages = [
+            { role: 'system', content: this.systemInstruction },
+            ...this.history.map(m => ({
+                role: m.role,
+                content: contentToOpenAIContent(m.content)
+            }))
+        ];
+        const response = await axios.post(`https://api-inference.huggingface.co/models/${model}/v1/chat/completions`, {
+            model,
+            messages,
+            max_tokens: this.maxTokens
+        }, {
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': 'application/json'
+            }
+        });
+        return response.data.choices[0].message.content;
+    }
+}
+
+module.exports = {
+    DEFAULT_GEMINI_MODEL,
+    ALL_PROVIDERS,
+    AgentProviderClient,
+    getProviderAttemptOrder,
+    getProviderModel,
+    _helpers: {
+        splitDataUri,
+        contentToGeminiParts,
+        contentToOpenAIContent,
+        contentToAnthropicContent,
+        contentToOllamaMessage
+    }
+};