@pheem49/mint 1.4.2 → 1.5.1

package/src/System/granular_automation.js

@@ -1,88 +1,157 @@
-const { exec } = require('child_process');
+const { execFile, spawnSync } = require('child_process');
 const { screen } = require('electron');
 
-/**
- * GranularAutomation handles low-level OS input via xdotool.
- * It uses a normalized coordinate system (0-1000).
- */
+function commandExists(command) {
+    const lookup = process.platform === 'win32' ? 'where' : 'which';
+    const result = spawnSync(lookup, [command], { encoding: 'utf8', shell: false });
+    return result.status === 0;
+}
+
+function run(command, args = []) {
+    return new Promise((resolve, reject) => {
+        execFile(command, args, (err, stdout, stderr) => {
+            if (err) {
+                err.stderr = stderr;
+                reject(err);
+                return;
+            }
+            resolve(stdout);
+        });
+    });
+}
+
+function unsupported(feature) {
+    throw new Error(`${feature} is not supported on ${process.platform} by the current input automation provider.`);
+}
+
+function escapePowerShellSingleQuoted(value) {
+    return String(value || '').replace(/'/g, "''");
+}
+
+function keyToMacKey(key) {
+    const value = String(key || '').trim();
+    const map = {
+        Enter: 'return',
+        Return: 'return',
+        Escape: 'escape',
+        Esc: 'escape',
+        Space: 'space',
+        Backspace: 'delete',
+        Delete: 'forward delete',
+        Tab: 'tab'
+    };
+    return map[value] || value;
+}
+
 class GranularAutomation {
     constructor() {
-        this.screenWidth = 1920; // Default fallback
+        this.screenWidth = 1920;
         this.screenHeight = 1080;
         this.updateScreenSize();
     }
 
     updateScreenSize() {
         try {
-            // In Electron main process, we can use the screen module
             const primaryDisplay = screen.getPrimaryDisplay();
             if (primaryDisplay && primaryDisplay.size) {
                 this.screenWidth = primaryDisplay.size.width;
                 this.screenHeight = primaryDisplay.size.height;
-                console.log(`[Automation] Screen detected: ${this.screenWidth}x${this.screenHeight}`);
             }
-        } catch (e) {
-            // Fallback for CLI or cases where screen module is unavailable
-            exec('xdpyinfo | grep dimensions', (err, stdout) => {
-                if (!err && stdout) {
-                    const match = stdout.match(/(\d+)x(\d+) pixels/);
-                    if (match) {
-                        this.screenWidth = parseInt(match[1]);
-                        this.screenHeight = parseInt(match[2]);
-                        console.log(`[Automation] Screen detected via xdpyinfo: ${this.screenWidth}x${this.screenHeight}`);
-                    }
-                }
-            });
+        } catch (_) {
+            // Electron screen can be unavailable in CLI-only contexts.
         }
     }
 
     scaleX(x) {
-        return Math.round((x / 1000) * this.screenWidth);
+        return Math.round((Number(x) / 1000) * this.screenWidth);
     }
 
     scaleY(y) {
-        return Math.round((y / 1000) * this.screenHeight);
+        return Math.round((Number(y) / 1000) * this.screenHeight);
     }
 
-    run(command) {
-        return new Promise((resolve, reject) => {
-            exec(command, (err, stdout, stderr) => {
-                if (err) {
-                    console.error(`[Automation] xdotool error: ${stderr}`);
-                    reject(err);
-                } else {
-                    resolve(stdout);
-                }
-            });
-        });
+    provider() {
+        if (process.platform === 'darwin') return macProvider;
+        if (process.platform === 'win32') return windowsProvider;
+        return linuxProvider;
     }
 
-    async mouseMove(x, y) {
-        const sx = this.scaleX(x);
-        const sy = this.scaleY(y);
-        console.log(`[Automation] Moving mouse to ${sx}, ${sy}`);
-        return this.run(`xdotool mousemove ${sx} ${sy}`);
+    mouseMove(x, y) {
+        return this.provider().mouseMove(this.scaleX(x), this.scaleY(y));
     }
 
-    async mouseClick(x, y, button = 1) {
-        const sx = this.scaleX(x);
-        const sy = this.scaleY(y);
-        console.log(`[Automation] Clicking ${button} at ${sx}, ${sy}`);
-        // move first then click to be safe
-        return this.run(`xdotool mousemove ${sx} ${sy} click ${button}`);
+    mouseClick(x, y, button = 1) {
+        return this.provider().mouseClick(this.scaleX(x), this.scaleY(y), button);
     }
 
-    async typeText(text) {
-        console.log(`[Automation] Typing: ${text}`);
-        // Escape double quotes for shell
-        const escaped = text.replace(/"/g, '\\"');
-        return this.run(`xdotool type "${escaped}"`);
+    typeText(text) {
+        return this.provider().typeText(String(text || ''));
     }
 
-    async keyTap(key) {
-        console.log(`[Automation] Key tap: ${key}`);
-        return this.run(`xdotool key "${key}"`);
+    keyTap(key) {
+        return this.provider().keyTap(String(key || ''));
     }
 }
 
-module.exports = new GranularAutomation();
+const linuxProvider = {
+    mouseMove: (x, y) => run('xdotool', ['mousemove', String(x), String(y)]),
+    mouseClick: (x, y, button = 1) => run('xdotool', ['mousemove', String(x), String(y), 'click', String(button)]),
+    typeText: (text) => run('xdotool', ['type', text]),
+    keyTap: (key) => run('xdotool', ['key', key])
+};
+
+const macProvider = {
+    mouseMove(x, y) {
+        if (!commandExists('cliclick')) return unsupported('Mouse move');
+        return run('cliclick', [`m:${x},${y}`]);
+    },
+    mouseClick(x, y) {
+        if (!commandExists('cliclick')) return unsupported('Mouse click');
+        return run('cliclick', [`c:${x},${y}`]);
+    },
+    typeText(text) {
+        if (commandExists('cliclick')) return run('cliclick', [`t:${text}`]);
+        return run('osascript', ['-e', `tell application "System Events" to keystroke ${JSON.stringify(text)}`]);
+    },
+    keyTap(key) {
+        const macKey = keyToMacKey(key);
+        if (commandExists('cliclick')) return run('cliclick', [`kp:${macKey}`]);
+        if (macKey.length === 1) {
+            return run('osascript', ['-e', `tell application "System Events" to keystroke ${JSON.stringify(macKey)}`]);
+        }
+        return unsupported('Special key tap without cliclick');
+    }
+};
+
+const windowsProvider = {
+    mouseMove(x, y) {
+        const script = `[void][Reflection.Assembly]::LoadWithPartialName('System.Windows.Forms'); [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})`;
+        return run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script]);
+    },
+    mouseClick(x, y, button = 1) {
+        const down = Number(button) === 2 ? '0x0008' : '0x0002';
+        const up = Number(button) === 2 ? '0x0010' : '0x0004';
+        const script = [
+            "Add-Type -MemberDefinition '[DllImport(\"user32.dll\")] public static extern bool SetCursorPos(int X,int Y); [DllImport(\"user32.dll\")] public static extern void mouse_event(int dwFlags,int dx,int dy,int dwData,int dwExtraInfo);' -Name NativeMouse -Namespace Mint;",
+            `[Mint.NativeMouse]::SetCursorPos(${x}, ${y}) | Out-Null;`,
+            `[Mint.NativeMouse]::mouse_event(${down},0,0,0,0);`,
+            `[Mint.NativeMouse]::mouse_event(${up},0,0,0,0);`
+        ].join(' ');
+        return run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script]);
+    },
+    typeText(text) {
+        const safe = escapePowerShellSingleQuoted(text);
+        const script = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${safe}')`;
+        return run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script]);
+    },
+    keyTap(key) {
+        const safe = escapePowerShellSingleQuoted(key);
+        const script = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('{${safe}}')`;
+        return run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script]);
+    }
+};
+
+const instance = new GranularAutomation();
+instance._providers = { linuxProvider, macProvider, windowsProvider, commandExists };
+
+module.exports = instance;

package/src/System/ipc_handlers.js

@@ -0,0 +1,238 @@
+function registerIpcHandlers({
+    app,
+    ipcMain,
+    shell,
+    clipboard,
+    windowManager,
+    proactiveLoop,
+    screenCapture,
+    services
+}) {
+    const {
+        handleChat,
+        resetChat,
+        getChatTranscript,
+        refreshApiKeyFromConfig,
+        getSystemInfo,
+        getWeather,
+        readConfig,
+        writeConfig,
+        parseCommand,
+        executeAction,
+        getGoogleTtsUrls,
+        customWorkflows
+    } = services;
+
+    ipcMain.handle('chat-message', async (event, message, base64Image = null, base64Audio = null) => {
+        try {
+            const rawResponse = await handleChat(message, base64Image, base64Audio);
+            const aiResponse = parseCommand(rawResponse);
+
+            if (aiResponse.action && aiResponse.action.type !== 'none') {
+                try {
+                    const actionResult = await executeAction(aiResponse.action, { clipboard });
+                    if (actionResult && typeof actionResult === 'string') {
+                        aiResponse.response += `\n\n${actionResult}`;
+                    }
+                } catch (err) {
+                    console.error("Action execution error:", err);
+                    aiResponse.response += "\n\n(Note: I tried to execute the action, but an error occurred.)";
+                }
+            }
+
+            return aiResponse;
+        } catch (error) {
+            console.error('Chat error:', error);
+            return { response: 'Error communicating with Gemini API. Check your console and API key.', action: { type: 'none' } };
+        }
+    });
+
+    ipcMain.on('close-window', () => {
+        const mainWindow = windowManager.getMainWindow();
+        if (mainWindow) mainWindow.hide();
+    });
+
+    ipcMain.on('minimize-window', () => {
+        const mainWindow = windowManager.getMainWindow();
+        if (mainWindow) mainWindow.minimize();
+    });
+
+    ipcMain.on('quit-app', () => {
+        app.isQuiting = true;
+        app.quit();
+    });
+
+    ipcMain.on('maximize-window', () => {
+        const mainWindow = windowManager.getMainWindow();
+        if (!mainWindow) return;
+        if (mainWindow.isMaximized()) {
+            mainWindow.unmaximize();
+        } else {
+            mainWindow.maximize();
+        }
+    });
+
+    ipcMain.handle('reset-chat', () => {
+        resetChat();
+        return { success: true };
+    });
+
+    ipcMain.handle('get-chat-history', () => getChatTranscript());
+
+    ipcMain.handle('open-settings', () => {
+        windowManager.createSettingsWindow();
+    });
+
+    ipcMain.handle('get-settings', () => readConfig());
+
+    ipcMain.handle('save-settings', (event, config) => {
+        console.log('[Settings] Saving new config. MCP Servers count:', Object.keys(config.mcpServers || {}).length);
+        const result = writeConfig(config);
+        refreshApiKeyFromConfig();
+
+        const mainWindow = windowManager.getMainWindow();
+        if (mainWindow && !mainWindow.isDestroyed()) {
+            mainWindow.webContents.send('settings-changed', config);
+        }
+
+        if (proactiveLoop.isRunning()) {
+            proactiveLoop.start(config.proactiveInterval);
+        }
+
+        if (config.enableCustomWorkflows !== false) {
+            customWorkflows.startMonitoring(mainWindow.webContents);
+        } else {
+            customWorkflows.stopMonitoring();
+        }
+
+        if (config.showDesktopWidget === false) {
+            windowManager.closeWidgetWindow();
+        } else {
+            windowManager.ensureWidgetWindow();
+        }
+
+        return result;
+    });
+
+    ipcMain.on('set-ai-state', (event, state) => {
+        const widgetWindow = windowManager.getWidgetWindow();
+        if (widgetWindow && !widgetWindow.isDestroyed()) {
+            widgetWindow.webContents.send('widget-state', state);
+        }
+    });
+
+    ipcMain.on('close-settings', () => {
+        const settingsWindow = windowManager.getSettingsWindow();
+        if (settingsWindow) settingsWindow.close();
+    });
+
+    ipcMain.handle('open-custom-workflows', () => {
+        customWorkflows.openConfigFile();
+    });
+
+    ipcMain.handle('reload-custom-workflows', () => {
+        customWorkflows.loadWorkflows();
+        return { success: true };
+    });
+
+    ipcMain.on('spotlight-close', () => {
+        const spotlightWindow = windowManager.getSpotlightWindow();
+        if (spotlightWindow) spotlightWindow.close();
+    });
+
+    ipcMain.on('spotlight-hide', () => {
+        const spotlightWindow = windowManager.getSpotlightWindow();
+        if (spotlightWindow) spotlightWindow.hide();
+    });
+
+    ipcMain.on('spotlight-submit', async (event, query) => {
+        console.log('[Spotlight] Submit:', query);
+        const spotlightWindow = windowManager.getSpotlightWindow();
+        if (spotlightWindow) spotlightWindow.hide();
+
+        const mainWindow = windowManager.getMainWindow();
+        if (mainWindow) {
+            mainWindow.show();
+            mainWindow.webContents.send('spotlight-to-chat', query);
+        }
+    });
+
+    ipcMain.on('spotlight-resize', (event, width, height) => {
+        const spotlightWindow = windowManager.getSpotlightWindow();
+        if (spotlightWindow) spotlightWindow.setSize(width, height);
+    });
+
+    ipcMain.handle('open-external', (event, url) => {
+        shell.openExternal(url);
+    });
+
+    ipcMain.handle('clipboard-read', () => clipboard.readText());
+
+    ipcMain.handle('clipboard-write', (event, text) => {
+        clipboard.writeText(text);
+        return { success: true };
+    });
+
+    ipcMain.handle('get-tts-urls', async (event, text) => {
+        try {
+            const isThai = /[\u0E00-\u0E7F]/.test(text);
+            return getGoogleTtsUrls(text, {
+                lang: isThai ? 'th' : 'en',
+                host: 'https://translate.google.com',
+            });
+        } catch (e) {
+            console.error("TTS Error:", e);
+            return [];
+        }
+    });
+
+    ipcMain.handle('get-system-info', async () => getSystemInfo());
+    ipcMain.handle('get-weather', async (event, city) => getWeather(city));
+
+    ipcMain.handle('start-screen-capture', () => screenCapture.startScreenCapture());
+    ipcMain.on('vision-selection', (event, base64Image) => screenCapture.handleSelection(base64Image));
+    ipcMain.on('vision-translate-start', (event, rect) => screenCapture.startLiveTranslate(rect));
+    ipcMain.on('vision-translate-stop', () => screenCapture.stopLiveTranslate());
+    ipcMain.on('vision-overlay-interactable', (event, isInteractable) => screenCapture.setOverlayInteractable(isInteractable));
+    ipcMain.on('vision-cancel', () => screenCapture.cancel());
+    ipcMain.handle('capture-silent-screen', () => screenCapture.captureSilentScreen());
+
+    ipcMain.on('toggle-proactive', (event, isOn) => {
+        if (isOn) {
+            proactiveLoop.start();
+        } else {
+            proactiveLoop.stop();
+        }
+    });
+
+    ipcMain.on('record-behavior', (event, contextDescription) => {
+        proactiveLoop.recordBehavior(contextDescription);
+    });
+
+    ipcMain.handle('execute-proactive-action', async (event, action) => {
+        if (!action || action.type === 'none') {
+            return { success: false, message: 'ไม่มี action ที่จะดำเนินการค่ะ' };
+        }
+        try {
+            const result = await executeAction(action, { clipboard });
+            const messages = {
+                open_url: `เปิดเว็บไซต์ให้แล้วค่ะ 🌐`,
+                open_app: `เปิดแอป ${action.target} ให้แล้วค่ะ 🚀`,
+                search: `ค้นหา "${action.target}" ให้แล้วค่ะ 🔍`,
+                web_automation: result || 'ดำเนินการเสร็จแล้วค่ะ ✅',
+                create_folder: `สร้างโฟลเดอร์ "${action.target}" แล้วค่ะ 📁`,
+                clipboard_write: `คัดลอกข้อความแล้วค่ะ 📋`,
+                learn_file: result || `เรียนรู้เอกสารเรียบร้อยค่ะ 📚`,
+            };
+            return {
+                success: true,
+                message: messages[action.type] || 'ดำเนินการเสร็จแล้วค่ะ ✅'
+            };
+        } catch (err) {
+            console.error('[ProactiveAction] Error:', err);
+            return { success: false, message: `เกิดข้อผิดพลาด: ${err.message}` };
+        }
+    });
+}
+
+module.exports = { registerIpcHandlers };

package/src/System/proactive_loop.js

@@ -0,0 +1,153 @@
+const { BrowserWindow, desktopCapturer, screen, powerMonitor } = require('electron');
+const path = require('path');
+
+const IDLE_THRESHOLD_SEC = 300;
+
+let proactiveEngine = null;
+function getProactiveEngine() {
+    if (!proactiveEngine) {
+        proactiveEngine = require('../AI_Brain/proactive_engine');
+    }
+    return proactiveEngine;
+}
+
+let behaviorMemory = null;
+function getBehaviorMemory() {
+    if (!behaviorMemory) {
+        behaviorMemory = require('../AI_Brain/behavior_memory');
+    }
+    return behaviorMemory;
+}
+
+function createProactiveLoop({ app, projectRoot, readConfig, getMainWindow }) {
+    let proactiveGlowWindow = null;
+    let proactiveIntervalHandle = null;
+    let idleWatcherHandle = null;
+
+    async function runProactiveCycle() {
+        const mainWindow = getMainWindow();
+        if (!mainWindow || mainWindow.isDestroyed()) return;
+
+        try {
+            showProactiveGlow();
+
+            const primaryDisplay = screen.getPrimaryDisplay();
+            const width = Math.floor(primaryDisplay.size.width * 0.5);
+            const height = Math.floor(primaryDisplay.size.height * 0.5);
+            const sources = await desktopCapturer.getSources({
+                types: ['screen'],
+                thumbnailSize: { width, height }
+            });
+
+            const primarySource = sources[0];
+            if (!primarySource || !primarySource.thumbnail) return;
+
+            const base64Image = primarySource.thumbnail.toJPEG(60).toString('base64');
+            const { analyzeAndSuggest } = getProactiveEngine();
+            const { recordBehavior, getBehaviorSummary } = getBehaviorMemory();
+            const result = await analyzeAndSuggest(base64Image, getBehaviorSummary());
+
+            if (result && result.message && Array.isArray(result.suggestions)) {
+                if (result.context) recordBehavior(result.context);
+
+                const currentMainWindow = getMainWindow();
+                if (currentMainWindow && !currentMainWindow.isDestroyed()) {
+                    currentMainWindow.webContents.send('proactive-suggestion', result);
+                }
+            }
+
+            hideProactiveGlow();
+        } catch (err) {
+            console.error('[Proactive] Cycle error:', err.message);
+            hideProactiveGlow();
+        }
+    }
+
+    function createProactiveGlowWindow() {
+        if (proactiveGlowWindow) return proactiveGlowWindow;
+        const { width, height } = screen.getPrimaryDisplay().bounds;
+
+        proactiveGlowWindow = new BrowserWindow({
+            width,
+            height,
+            x: 0,
+            y: 0,
+            frame: false,
+            transparent: true,
+            alwaysOnTop: true,
+            skipTaskbar: true,
+            focusable: false,
+            show: false,
+            webPreferences: {
+                nodeIntegration: false,
+                contextIsolation: true
+            }
+        });
+
+        proactiveGlowWindow.setIgnoreMouseEvents(true);
+        proactiveGlowWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
+        proactiveGlowWindow.setAlwaysOnTop(true, 'screen-saver');
+        proactiveGlowWindow.loadFile(path.join(projectRoot, 'src/UI/proactive-glow.html'));
+        proactiveGlowWindow.on('closed', () => { proactiveGlowWindow = null; });
+        return proactiveGlowWindow;
+    }
+
+    function showProactiveGlow() {
+        if (!proactiveGlowWindow) createProactiveGlowWindow();
+        if (proactiveGlowWindow) proactiveGlowWindow.showInactive();
+    }
+
+    function hideProactiveGlow() {
+        if (proactiveGlowWindow && !proactiveGlowWindow.isDestroyed()) {
+            proactiveGlowWindow.hide();
+        }
+    }
+
+    function start(intervalSec) {
+        stop();
+        const cfg = readConfig();
+        const ms = (intervalSec || cfg.proactiveInterval || 60) * 1000;
+        console.log(`[Proactive] Starting loop — interval: ${ms / 1000}s`);
+        proactiveIntervalHandle = setInterval(runProactiveCycle, ms);
+    }
+
+    function stop() {
+        if (proactiveIntervalHandle) {
+            clearInterval(proactiveIntervalHandle);
+            proactiveIntervalHandle = null;
+            console.log('[Proactive] Stopped proactive loop.');
+        }
+    }
+
+    function startIdleWatcher() {
+        if (idleWatcherHandle) return;
+        idleWatcherHandle = setInterval(() => {
+            if (!proactiveIntervalHandle) return;
+            if (!app.isReady()) return;
+
+            const idleSec = powerMonitor.getSystemIdleTime();
+            if (idleSec < IDLE_THRESHOLD_SEC) return;
+
+            console.log(`[System Idle] User idle for ${idleSec}s. Pausing Proactive loop to save resources.`);
+            stop();
+
+            const resumeChecker = setInterval(() => {
+                if (powerMonitor.getSystemIdleTime() < 10) {
+                    console.log('[System Idle] User returned. Resuming Proactive loop.');
+                    clearInterval(resumeChecker);
+                    start();
+                }
+            }, 5000);
+        }, 60000);
+    }
+
+    return {
+        start,
+        stop,
+        startIdleWatcher,
+        isRunning: () => Boolean(proactiveIntervalHandle),
+        recordBehavior: (...args) => getBehaviorMemory().recordBehavior(...args)
+    };
+}
+
+module.exports = { createProactiveLoop };