@pheem49/mint 1.4.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/GUIDE_TH.md +113 -0
  2. package/README.md +214 -142
  3. package/assets/CLI_Screen.png +0 -0
  4. package/docs/assets/CLI_Screen.png +0 -0
  5. package/docs/guide.html +632 -0
  6. package/docs/index.html +5 -4
  7. package/main.js +66 -894
  8. package/mint-cli-logic.js +15 -8
  9. package/mint-cli.js +305 -195
  10. package/package.json +12 -4
  11. package/src/AI_Brain/Gemini_API.js +77 -20
  12. package/src/AI_Brain/agent_orchestrator.js +6 -6
  13. package/src/AI_Brain/autonomous_brain.js +10 -0
  14. package/src/AI_Brain/behavior_memory.js +26 -5
  15. package/src/AI_Brain/headless_agent.js +4 -0
  16. package/src/AI_Brain/knowledge_base.js +61 -8
  17. package/src/AI_Brain/memory_store.js +55 -7
  18. package/src/Automation_Layer/file_operations.js +14 -3
  19. package/src/CLI/chat_router.js +21 -7
  20. package/src/CLI/chat_ui.js +264 -710
  21. package/src/CLI/code_agent.js +370 -124
  22. package/src/CLI/gmail_auth.js +210 -0
  23. package/src/CLI/list_features.js +5 -1
  24. package/src/CLI/onboarding.js +307 -55
  25. package/src/CLI/updater.js +208 -0
  26. package/src/Channels/brave_search_bridge.js +35 -0
  27. package/src/Channels/discord_bridge.js +68 -0
  28. package/src/Channels/google_search_bridge.js +38 -0
  29. package/src/Channels/line_bridge.js +60 -0
  30. package/src/Channels/slack_bridge.js +53 -0
  31. package/src/Channels/telegram_bridge.js +49 -0
  32. package/src/Channels/whatsapp_bridge.js +55 -0
  33. package/src/Command_Parser/parser.js +12 -1
  34. package/src/Plugins/gmail.js +251 -0
  35. package/src/Plugins/google_calendar.js +245 -19
  36. package/src/Plugins/notion.js +256 -0
  37. package/src/System/action_executor.js +129 -0
  38. package/src/System/bridge_manager.js +76 -0
  39. package/src/System/chat_history_manager.js +23 -5
  40. package/src/System/config_manager.js +41 -7
  41. package/src/System/custom_workflows.js +31 -2
  42. package/src/System/google_tts_urls.js +51 -0
  43. package/src/System/ipc_handlers.js +238 -0
  44. package/src/System/proactive_loop.js +137 -0
  45. package/src/System/safety_manager.js +165 -0
  46. package/src/System/screen_capture.js +175 -0
  47. package/src/System/task_manager.js +15 -5
  48. package/src/System/window_manager.js +210 -0
  49. package/src/UI/renderer.js +33 -7
  50. package/src/UI/settings.html +24 -0
  51. package/src/UI/settings.js +14 -4
  52. package/src/UI/styles.css +14 -1
  53. package/tests/action_executor_safety.test.js +67 -0
  54. package/tests/gmail.test.js +135 -0
  55. package/tests/gmail_auth.test.js +129 -0
  56. package/tests/google_calendar.test.js +113 -0
  57. package/tests/google_tts_urls.test.js +24 -0
  58. package/tests/notion.test.js +121 -0
  59. package/tests/provider_routing.test.js +17 -1
  60. package/tests/safety_manager.test.js +40 -0
  61. package/tests/updater.test.js +32 -0
package/main.js CHANGED
@@ -1,360 +1,82 @@
1
- const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard, Tray, Menu, nativeImage, desktopCapturer, screen, powerMonitor } = require('electron');
2
- const path = require('path');
1
+ const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard } = require('electron');
3
2
  require('dotenv').config();
4
3
 
5
4
  const { handleChat, resetChat, getChatTranscript, translateImageContent, refreshApiKeyFromConfig } = require('./src/AI_Brain/Gemini_API');
6
- const { openApp } = require('./src/Automation_Layer/open_app');
7
- const { openWebsite, openSearch } = require('./src/Automation_Layer/open_website');
8
- const { performWebAutomation } = require('./src/Automation_Layer/browser_automation');
9
- const { createFolder, openFile, deleteFile, findPath } = require('./src/Automation_Layer/file_operations');
10
5
  const { getSystemInfo, getWeather } = require('./src/System/system_info');
11
6
  const { readConfig, writeConfig } = require('./src/System/config_manager');
12
7
  const { parseCommand } = require('./src/Command_Parser/parser');
13
- const pluginManager = require('./src/Plugins/plugin_manager');
14
- const { analyzeAndSuggest } = require('./src/AI_Brain/proactive_engine');
15
- const { recordBehavior, getBehaviorSummary } = require('./src/AI_Brain/behavior_memory');
16
- const { indexFile, indexFolder } = require('./src/AI_Brain/knowledge_base');
8
+ const { executeAction } = require('./src/System/action_executor');
9
+ const { getGoogleTtsUrls } = require('./src/System/google_tts_urls');
10
+ const { createWindowManager } = require('./src/System/window_manager');
11
+ const { createProactiveLoop } = require('./src/System/proactive_loop');
12
+ const { createScreenCaptureController } = require('./src/System/screen_capture');
13
+ const { registerIpcHandlers } = require('./src/System/ipc_handlers');
17
14
 
18
- const SystemAutomation = require('./src/System/system_automation');
19
15
  const systemEvents = require('./src/System/system_events');
20
16
  const customWorkflows = require('./src/System/custom_workflows');
21
17
  const mcpManager = require('./src/Plugins/mcp_manager');
22
- const granularAutomation = require('./src/System/granular_automation');
23
- const googleTTS = require('google-tts-api');
24
18
 
25
- let mainWindow;
26
- let settingsWindow = null;
27
- let screenPickerWindow = null;
28
- let spotlightWindow = null;
29
- let widgetWindow = null;
30
- let proactiveGlowWindow = null;
31
- let tray = null;
32
-
33
- // =====================
34
- // Proactive Loop
35
- // =====================
36
- let proactiveIntervalHandle = null;
37
-
38
- async function runProactiveCycle() {
39
- if (!mainWindow || mainWindow.isDestroyed()) return;
40
- try {
41
- // Show Glow
42
- showProactiveGlow();
43
-
44
- // Silent screen capture
45
- const primaryDisplay = screen.getPrimaryDisplay();
46
- // Downscale to 50% for performance
47
- const width = Math.floor(primaryDisplay.size.width * 0.5);
48
- const height = Math.floor(primaryDisplay.size.height * 0.5);
49
-
50
- const sources = await desktopCapturer.getSources({
51
- types: ['screen'],
52
- thumbnailSize: { width, height }
53
- });
54
- const primarySource = sources[0];
55
- if (!primarySource || !primarySource.thumbnail) return;
56
-
57
- // Compress image to JPEG to save huge base64 payload size and memory
58
- // .toJPEG(quality) where quality is 0-100
59
- const base64Image = primarySource.thumbnail.toJPEG(60).toString('base64');
60
- const behaviorSummary = getBehaviorSummary();
61
-
62
- const result = await analyzeAndSuggest(base64Image, behaviorSummary);
63
-
64
- if (result && result.message && Array.isArray(result.suggestions)) {
65
- // Record the observed context into behavior memory
66
- if (result.context) recordBehavior(result.context);
67
-
68
- // Push suggestion to the renderer
69
- if (mainWindow && !mainWindow.isDestroyed()) {
70
- mainWindow.webContents.send('proactive-suggestion', result);
71
- }
72
- }
73
-
74
- // Hide Glow after analysis
75
- hideProactiveGlow();
76
- } catch (err) {
77
- console.error('[Proactive] Cycle error:', err.message);
78
- hideProactiveGlow();
79
- }
80
- }
81
-
82
- function createProactiveGlowWindow() {
83
- if (proactiveGlowWindow) return;
84
- const { width, height } = screen.getPrimaryDisplay().bounds;
85
-
86
- proactiveGlowWindow = new BrowserWindow({
87
- width, height,
88
- x: 0, y: 0,
89
- frame: false,
90
- transparent: true,
91
- alwaysOnTop: true,
92
- skipTaskbar: true,
93
- focusable: false,
94
- show: false,
95
- webPreferences: {
96
- nodeIntegration: false,
97
- contextIsolation: true
98
- }
99
- });
100
-
101
- proactiveGlowWindow.setIgnoreMouseEvents(true);
102
- proactiveGlowWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
103
- proactiveGlowWindow.setAlwaysOnTop(true, 'screen-saver');
104
- proactiveGlowWindow.loadFile('src/UI/proactive-glow.html');
105
-
106
- proactiveGlowWindow.on('closed', () => { proactiveGlowWindow = null; });
107
- }
108
-
109
- function showProactiveGlow() {
110
- if (!proactiveGlowWindow) createProactiveGlowWindow();
111
- if (proactiveGlowWindow) proactiveGlowWindow.showInactive();
112
- }
113
-
114
- function hideProactiveGlow() {
115
- if (proactiveGlowWindow && !proactiveGlowWindow.isDestroyed()) {
116
- proactiveGlowWindow.hide();
117
- }
118
- }
119
-
120
- function startProactiveLoop(intervalSec) {
121
- // Stop existing loop first
122
- if (proactiveIntervalHandle) {
123
- clearInterval(proactiveIntervalHandle);
124
- proactiveIntervalHandle = null;
125
- }
126
- // Read from config if not provided
127
- const cfg = readConfig();
128
- const ms = (intervalSec || cfg.proactiveInterval || 60) * 1000;
129
- console.log(`[Proactive] Starting loop — interval: ${ms / 1000}s`);
130
- proactiveIntervalHandle = setInterval(runProactiveCycle, ms);
131
- }
132
-
133
- function stopProactiveLoop() {
134
- if (proactiveIntervalHandle) {
135
- clearInterval(proactiveIntervalHandle);
136
- proactiveIntervalHandle = null;
137
- console.log('[Proactive] Stopped proactive loop.');
138
- }
139
- }
140
-
141
- // Check idle state every minute to pause/resume background tasks
142
- const IDLE_THRESHOLD_SEC = 300; // 5 minutes
143
- setInterval(() => {
144
- if (!proactiveIntervalHandle) return; // Only manage if it's supposed to be active (Smart Context is ON)
145
-
146
- // powerMonitor.getSystemIdleTime() is available after app is ready
147
- if (app.isReady()) {
148
- const idleSec = powerMonitor.getSystemIdleTime();
149
- if (idleSec >= IDLE_THRESHOLD_SEC) {
150
- console.log(`[System Idle] User idle for ${idleSec}s. Pausing Proactive loop to save resources.`);
151
- stopProactiveLoop();
152
-
153
- // Wait for user to come back
154
- const resumeChecker = setInterval(() => {
155
- if (powerMonitor.getSystemIdleTime() < 10) {
156
- console.log('[System Idle] User returned. Resuming Proactive loop.');
157
- clearInterval(resumeChecker);
158
- startProactiveLoop();
159
- }
160
- }, 5000);
161
- }
162
- }
163
- }, 60000);
164
-
165
- function createWindow() {
166
- mainWindow = new BrowserWindow({
167
- width: 600,
168
- height: 800,
169
- icon: path.join(__dirname, 'assets', 'icon.png'),
170
- webPreferences: {
171
- preload: path.join(__dirname, 'preload.js'),
172
- nodeIntegration: false,
173
- contextIsolation: true,
174
- },
175
- frame: false,
176
- transparent: true,
177
- resizable: true,
178
- show: false
179
- });
180
-
181
- mainWindow.loadFile('src/UI/index.html');
182
-
183
- mainWindow.on('ready-to-show', () => {
184
- mainWindow.show();
185
- });
186
-
187
- mainWindow.on('close', function (event) {
188
- if (!app.isQuiting) {
189
- event.preventDefault();
190
- mainWindow.hide();
191
- }
192
- return false;
193
- });
194
-
195
- mainWindow.on('focus', () => {
196
- // clearFloatingUnread(); // Disabled
197
- });
198
- }
199
-
200
- function createTray() {
201
- const iconPath = path.join(__dirname, 'assets', 'icon.png');
202
- let icon = nativeImage.createFromPath(iconPath);
203
- // Optional: resize it for the tray if needed, but Electron handles this natively on many OSs
204
- icon = icon.resize({ width: 16, height: 16 });
205
-
206
- tray = new Tray(icon);
207
- tray.setToolTip('Mint AI Assistant');
208
-
209
- const contextMenu = Menu.buildFromTemplate([
210
- { label: 'Show App', click: () => { if (mainWindow) mainWindow.show(); } },
211
- { label: 'Settings', click: () => { createSettingsWindow(); } },
212
- { type: 'separator' },
213
- { label: 'Quit', click: () => {
214
- app.isQuiting = true;
215
- app.quit();
216
- }}
217
- ]);
218
-
219
- tray.setContextMenu(contextMenu);
220
-
221
- tray.on('click', () => {
222
- if (mainWindow) {
223
- if (mainWindow.isVisible()) {
224
- mainWindow.hide();
225
- } else {
226
- mainWindow.show();
227
- }
228
- }
229
- });
230
- }
231
-
232
- function createSettingsWindow() {
233
- if (settingsWindow) {
234
- settingsWindow.focus();
235
- return;
236
- }
237
- settingsWindow = new BrowserWindow({
238
- width: 720,
239
- height: 620,
240
- minWidth: 640,
241
- minHeight: 560,
242
- icon: path.join(__dirname, 'assets', 'icon.png'),
243
- webPreferences: {
244
- preload: path.join(__dirname, 'preload-settings.js'),
245
- nodeIntegration: false,
246
- contextIsolation: true,
247
- },
248
- frame: false,
249
- transparent: true,
250
- resizable: true,
251
- parent: mainWindow,
252
- });
253
- settingsWindow.loadFile('src/UI/settings.html');
254
- settingsWindow.on('closed', () => { settingsWindow = null; });
255
- }
256
-
257
- function createSpotlightWindow() {
258
- if (spotlightWindow) {
259
- spotlightWindow.show();
260
- return;
19
+ const projectRoot = __dirname;
20
+ const windowManager = createWindowManager(projectRoot);
21
+ const proactiveLoop = createProactiveLoop({
22
+ app,
23
+ projectRoot,
24
+ readConfig,
25
+ getMainWindow: windowManager.getMainWindow
26
+ });
27
+ const screenCapture = createScreenCaptureController({
28
+ projectRoot,
29
+ translateImageContent,
30
+ getMainWindow: windowManager.getMainWindow
31
+ });
32
+
33
+ registerIpcHandlers({
34
+ app,
35
+ ipcMain,
36
+ shell,
37
+ clipboard,
38
+ windowManager,
39
+ proactiveLoop,
40
+ screenCapture,
41
+ services: {
42
+ handleChat,
43
+ resetChat,
44
+ getChatTranscript,
45
+ refreshApiKeyFromConfig,
46
+ getSystemInfo,
47
+ getWeather,
48
+ readConfig,
49
+ writeConfig,
50
+ parseCommand,
51
+ executeAction,
52
+ getGoogleTtsUrls,
53
+ customWorkflows
261
54
  }
262
-
263
- const { width: screenWidth, height: screenHeight } = screen.getPrimaryDisplay().workAreaSize;
264
- const windowWidth = 600;
265
- const windowHeight = 80; // Starts small, expands if results show
266
-
267
- spotlightWindow = new BrowserWindow({
268
- width: windowWidth,
269
- height: windowHeight,
270
- x: Math.floor((screenWidth - windowWidth) / 2),
271
- y: Math.floor(screenHeight * 0.25), // 25% from top
272
- frame: false,
273
- transparent: true,
274
- alwaysOnTop: true,
275
- skipTaskbar: true,
276
- show: false,
277
- webPreferences: {
278
- preload: path.join(__dirname, 'src/UI/preload-spotlight.js'),
279
- nodeIntegration: false,
280
- contextIsolation: true,
281
- }
282
- });
283
-
284
- spotlightWindow.loadFile('src/UI/spotlight.html');
285
-
286
- spotlightWindow.on('blur', () => {
287
- spotlightWindow.hide();
288
- });
289
-
290
- spotlightWindow.on('closed', () => {
291
- spotlightWindow = null;
292
- });
293
- }
294
-
295
- // Floating window logic removed
296
-
297
- function createWidgetWindow() {
298
- if (widgetWindow) return;
299
-
300
- widgetWindow = new BrowserWindow({
301
- width: 150,
302
- height: 150,
303
- frame: false,
304
- transparent: true,
305
- resizable: false,
306
- alwaysOnTop: true,
307
- skipTaskbar: true,
308
- show: true,
309
- webPreferences: {
310
- preload: require('path').join(__dirname, 'src/UI/preload-widget.js'),
311
- nodeIntegration: false,
312
- contextIsolation: true
313
- }
314
- });
315
-
316
- widgetWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
317
- widgetWindow.setAlwaysOnTop(true, 'floating');
318
-
319
- // Position Top-Right loosely
320
- try {
321
- const primaryDisplay = require('electron').screen.getPrimaryDisplay();
322
- const { width, height, x, y } = primaryDisplay.workArea;
323
- widgetWindow.setPosition(x + width - 150 - 40, y + 40);
324
- } catch (e) {}
325
-
326
- widgetWindow.loadFile('src/UI/widget.html');
327
-
328
- widgetWindow.on('closed', () => {
329
- widgetWindow = null;
330
- });
331
- }
332
-
333
- // Floating unread logic removed
55
+ });
334
56
 
335
57
  app.whenReady().then(() => {
336
58
  const config = readConfig();
337
- createWindow();
338
- createTray();
339
- // createFloatingWindow(); // Removed
340
-
341
- // Only show AI widget if enabled in settings
59
+ const mainWindow = windowManager.createMainWindow();
60
+ windowManager.createTray();
61
+
342
62
  if (config.showDesktopWidget !== false) {
343
- createWidgetWindow();
63
+ windowManager.createWidgetWindow();
344
64
  }
345
65
 
346
- // Initialize MCP Servers
347
66
  mcpManager.init().catch(err => console.error('[MCP] Init Error:', err));
348
-
349
- // Start monitoring system events (battery, wifi, etc.)
67
+
68
+ const bridgeManager = require('./src/System/bridge_manager');
69
+ bridgeManager.init().catch(err => console.error('[BridgeManager] Init Error:', err));
70
+
350
71
  systemEvents.startMonitoring();
351
72
  if (config.enableCustomWorkflows !== false) {
352
73
  customWorkflows.startMonitoring(mainWindow.webContents);
353
74
  }
354
75
 
355
76
  systemEvents.on('low-battery', (level) => {
356
- if (mainWindow && !mainWindow.isDestroyed()) {
357
- mainWindow.webContents.send('proactive-notification', {
77
+ const currentMainWindow = windowManager.getMainWindow();
78
+ if (currentMainWindow && !currentMainWindow.isDestroyed()) {
79
+ currentMainWindow.webContents.send('proactive-notification', {
358
80
  message: `⚠️ Battery is low (${level}%). Please plug in your charger. ✨`,
359
81
  type: 'warning'
360
82
  });
@@ -362,580 +84,30 @@ app.whenReady().then(() => {
362
84
  });
363
85
 
364
86
  systemEvents.on('connection-change', (isOnline) => {
365
- if (mainWindow && !mainWindow.isDestroyed()) {
87
+ const currentMainWindow = windowManager.getMainWindow();
88
+ if (currentMainWindow && !currentMainWindow.isDestroyed()) {
366
89
  const msg = isOnline ? '✅ Internet connection restored. ✨' : '❌ Internet connection lost.';
367
- mainWindow.webContents.send('proactive-notification', { message: msg, type: 'info' });
90
+ currentMainWindow.webContents.send('proactive-notification', { message: msg, type: 'info' });
368
91
  }
369
92
  });
370
93
 
371
- globalShortcut.register('CommandOrControl+Shift+Space', () => {
372
- if (mainWindow) {
373
- if (mainWindow.isVisible()) {
374
- mainWindow.hide();
375
- } else {
376
- mainWindow.show();
377
- }
378
- }
379
- });
94
+ globalShortcut.register('CommandOrControl+Shift+Space', windowManager.toggleMainWindow);
95
+ globalShortcut.register('Alt+Space', windowManager.toggleSpotlightWindow);
380
96
 
381
- globalShortcut.register('Alt+Space', () => {
382
- if (spotlightWindow && spotlightWindow.isVisible()) {
383
- spotlightWindow.hide();
384
- } else {
385
- createSpotlightWindow();
386
- spotlightWindow.show();
387
- }
388
- });
97
+ proactiveLoop.startIdleWatcher();
389
98
 
390
- app.on('activate', function () {
391
- if (BrowserWindow.getAllWindows().length === 0) createWindow();
99
+ app.on('activate', () => {
100
+ if (BrowserWindow.getAllWindows().length === 0) {
101
+ windowManager.createMainWindow();
102
+ }
392
103
  });
393
104
  });
394
105
 
395
- app.on('window-all-closed', (e) => {
396
- e.preventDefault();
106
+ app.on('window-all-closed', (event) => {
107
+ event.preventDefault();
397
108
  });
398
109
 
399
110
  app.on('will-quit', () => {
400
111
  globalShortcut.unregisterAll();
401
112
  mcpManager.shutdown();
402
113
  });
403
-
404
- // =====================
405
- // IPC Handlers — Chat
406
- // =====================
407
- ipcMain.handle('chat-message', async (event, message, base64Image = null, base64Audio = null) => {
408
- try {
409
- const rawResponse = await handleChat(message, base64Image, base64Audio);
410
- const aiResponse = parseCommand(rawResponse);
411
-
412
- if (aiResponse.action && aiResponse.action.type !== 'none') {
413
- try {
414
- const actionResult = await executeAction(aiResponse.action);
415
- // If the action returned a string result (e.g. from Web Automation), append it.
416
- if (actionResult && typeof actionResult === 'string') {
417
- aiResponse.response += `\n\n${actionResult}`;
418
- }
419
- } catch (err) {
420
- console.error("Action execution error:", err);
421
- aiResponse.response += "\n\n(Note: I tried to execute the action, but an error occurred.)";
422
- }
423
- }
424
-
425
- return aiResponse;
426
- } catch (error) {
427
- console.error('Chat error:', error);
428
- return { response: 'Error communicating with Gemini API. Check your console and API key.', action: { type: 'none' } };
429
- }
430
- });
431
-
432
- ipcMain.on('close-window', (event) => {
433
- if (mainWindow) mainWindow.hide();
434
- });
435
-
436
- ipcMain.on('minimize-window', (event) => {
437
- if (mainWindow) mainWindow.minimize();
438
- });
439
-
440
- ipcMain.on('quit-app', (event) => {
441
- app.isQuiting = true;
442
- app.quit();
443
- });
444
-
445
- ipcMain.on('maximize-window', (event) => {
446
- if (mainWindow) {
447
- if (mainWindow.isMaximized()) {
448
- mainWindow.unmaximize();
449
- } else {
450
- mainWindow.maximize();
451
- }
452
- }
453
- });
454
-
455
- ipcMain.handle('reset-chat', () => {
456
- resetChat();
457
- return { success: true };
458
- });
459
-
460
- ipcMain.handle('get-chat-history', () => {
461
- return getChatTranscript();
462
- });
463
-
464
- // =====================
465
- // IPC Handlers — Settings
466
- // =====================
467
- ipcMain.handle('open-settings', () => {
468
- createSettingsWindow();
469
- });
470
-
471
- ipcMain.handle('get-settings', () => {
472
- return readConfig();
473
- });
474
-
475
- ipcMain.handle('save-settings', (event, config) => {
476
- console.log('[Settings] Saving new config. MCP Servers count:', Object.keys(config.mcpServers || {}).length);
477
- const result = writeConfig(config);
478
- // Refresh API key if user updated it in settings
479
- refreshApiKeyFromConfig();
480
- // 🔔 Notify main chat window immediately
481
- if (mainWindow && !mainWindow.isDestroyed()) {
482
- mainWindow.webContents.send('settings-changed', config);
483
- }
484
- // Restart proactive loop with new interval if it's running
485
- if (proactiveIntervalHandle) {
486
- startProactiveLoop(config.proactiveInterval);
487
- }
488
- if (config.enableCustomWorkflows !== false) {
489
- customWorkflows.startMonitoring(mainWindow.webContents);
490
- } else {
491
- customWorkflows.stopMonitoring();
492
- }
493
-
494
- // Toggle Desktop Widget
495
- if (config.showDesktopWidget === false) {
496
- if (widgetWindow && !widgetWindow.isDestroyed()) {
497
- widgetWindow.close();
498
- widgetWindow = null;
499
- }
500
- } else {
501
- if (!widgetWindow || widgetWindow.isDestroyed()) {
502
- createWidgetWindow();
503
- }
504
- }
505
-
506
- return result;
507
- });
508
-
509
- ipcMain.on('set-ai-state', (event, state) => {
510
- if (widgetWindow && !widgetWindow.isDestroyed()) {
511
- widgetWindow.webContents.send('widget-state', state);
512
- }
513
- });
514
-
515
- ipcMain.on('close-settings', () => {
516
- if (settingsWindow) settingsWindow.close();
517
- });
518
-
519
- ipcMain.on('quit-app', () => {
520
- app.isQuiting = true;
521
- app.quit();
522
- });
523
-
524
- ipcMain.handle('open-custom-workflows', () => {
525
- customWorkflows.openConfigFile();
526
- });
527
-
528
- ipcMain.handle('reload-custom-workflows', () => {
529
- customWorkflows.loadWorkflows();
530
- return { success: true };
531
- });
532
-
533
- // =====================
534
- // IPC Handlers — Spotlight
535
- // =====================
536
- ipcMain.on('spotlight-close', () => {
537
- if (spotlightWindow) spotlightWindow.close();
538
- });
539
-
540
- ipcMain.on('spotlight-hide', () => {
541
- if (spotlightWindow) spotlightWindow.hide();
542
- });
543
-
544
- ipcMain.on('spotlight-submit', async (event, query) => {
545
- console.log('[Spotlight] Submit:', query);
546
- if (spotlightWindow) spotlightWindow.hide();
547
-
548
- // Show main window and send the message
549
- if (mainWindow) {
550
- mainWindow.show();
551
- // We can either just show it or actually trigger the chat
552
- mainWindow.webContents.send('spotlight-to-chat', query);
553
- }
554
- });
555
-
556
- ipcMain.on('spotlight-resize', (event, width, height) => {
557
- if (spotlightWindow) {
558
- spotlightWindow.setSize(width, height);
559
- }
560
- });
561
-
562
- // Floating IPC removed
563
-
564
- ipcMain.handle('open-external', (event, url) => {
565
- shell.openExternal(url);
566
- });
567
-
568
- // =====================
569
- // IPC Handlers — Clipboard
570
- // =====================
571
- ipcMain.handle('clipboard-read', () => {
572
- return clipboard.readText();
573
- });
574
-
575
- ipcMain.handle('clipboard-write', (event, text) => {
576
- clipboard.writeText(text);
577
- return { success: true };
578
- });
579
-
580
- // =====================
581
- // IPC Handlers — TTS
582
- // =====================
583
- ipcMain.handle('get-tts-urls', async (event, text) => {
584
- try {
585
- const isThai = /[\u0E00-\u0E7F]/.test(text);
586
- const lang = isThai ? 'th' : 'en';
587
-
588
- const results = googleTTS.getAllAudioUrls(text, {
589
- lang: lang,
590
- slow: false,
591
- host: 'https://translate.google.com',
592
- splitPunct: ',.?!;:',
593
- });
594
- return results;
595
- } catch (e) {
596
- console.error("TTS Error:", e);
597
- return [];
598
- }
599
- });
600
-
601
- // =====================
602
- // IPC Handlers — System Info
603
- // =====================
604
- ipcMain.handle('get-system-info', async () => {
605
- return getSystemInfo();
606
- });
607
-
608
- ipcMain.handle('get-weather', async (event, city) => {
609
- return getWeather(city);
610
- });
611
-
612
- // =====================
613
- // IPC Handlers — Screen Vision
614
- // =====================
615
- ipcMain.handle('start-screen-capture', async () => {
616
- if (screenPickerWindow) return; // Prevent multiple windows
617
-
618
- try {
619
- // Capture primary display
620
- const primaryDisplay = screen.getPrimaryDisplay();
621
- const { width, height } = primaryDisplay.size;
622
-
623
- const sources = await desktopCapturer.getSources({
624
- types: ['screen'],
625
- thumbnailSize: { width, height }
626
- });
627
-
628
- // Find the full primary screen (usually "Screen 1" or "Entire screen")
629
- const primarySource = sources[0]; // Assuming single/primary monitor is the first
630
-
631
- // Create transparent, borderless screen picker window
632
- screenPickerWindow = new BrowserWindow({
633
- width, height,
634
- x: primaryDisplay.bounds.x, y: primaryDisplay.bounds.y,
635
- fullscreen: true,
636
- transparent: true,
637
- frame: false,
638
- alwaysOnTop: true,
639
- skipTaskbar: true,
640
- webPreferences: {
641
- preload: path.join(__dirname, 'preload-picker.js'),
642
- nodeIntegration: false,
643
- contextIsolation: true
644
- }
645
- });
646
-
647
- await screenPickerWindow.loadFile('src/UI/screenPicker.html');
648
-
649
- // Ensure image data isn't missing
650
- if(primarySource && primarySource.thumbnail) {
651
- screenPickerWindow.webContents.send('screenshot-data', primarySource.thumbnail.toDataURL());
652
- }
653
-
654
- screenPickerWindow.on('closed', () => { screenPickerWindow = null; });
655
- } catch (err) {
656
- console.error("Error starting screen capture:", err);
657
- }
658
- });
659
-
660
- // Received selection from the picker
661
- ipcMain.on('vision-selection', (event, base64Image) => {
662
- if (screenPickerWindow) screenPickerWindow.close();
663
-
664
- // Relay image back to main window interface
665
- if (mainWindow) {
666
- mainWindow.webContents.send('vision-ready', base64Image);
667
- mainWindow.show(); // Bring chat back to focus
668
- }
669
- });
670
-
671
- let translateIntervalHandle = null;
672
- let isTranslateRequestInFlight = false;
673
- let translateCooldownUntil = 0;
674
- const TRANSLATE_REFRESH_MS = 3000;
675
- const TRANSLATE_FAILURE_COOLDOWN_MS = 15000;
676
-
677
- // Start Continuous Live Translate
678
- ipcMain.on('vision-translate-start', async (event, rect) => {
679
- if (!screenPickerWindow) return;
680
-
681
- screenPickerWindow.setIgnoreMouseEvents(true, { forward: true });
682
- isTranslateRequestInFlight = false;
683
- translateCooldownUntil = 0;
684
-
685
- // Stop any existing loop
686
- if (translateIntervalHandle) clearInterval(translateIntervalHandle);
687
-
688
- // Initial capture immediately
689
- captureAndTranslate(rect);
690
-
691
- // Then start ticking every 3 seconds
692
- translateIntervalHandle = setInterval(() => {
693
- captureAndTranslate(rect);
694
- }, TRANSLATE_REFRESH_MS);
695
- });
696
-
697
- // Stop Continuous Live Translate
698
- ipcMain.on('vision-translate-stop', () => {
699
- if (translateIntervalHandle) {
700
- clearInterval(translateIntervalHandle);
701
- translateIntervalHandle = null;
702
- }
703
- if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
704
- screenPickerWindow.setIgnoreMouseEvents(false);
705
- }
706
- isTranslateRequestInFlight = false;
707
- translateCooldownUntil = 0;
708
- });
709
-
710
- ipcMain.on('vision-overlay-interactable', (event, isInteractable) => {
711
- if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
712
- screenPickerWindow.setIgnoreMouseEvents(!isInteractable, { forward: true });
713
- });
714
-
715
- async function captureAndTranslate(rect) {
716
- if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
717
- if (isTranslateRequestInFlight) return;
718
- if (Date.now() < translateCooldownUntil) return;
719
-
720
- try {
721
- isTranslateRequestInFlight = true;
722
- const primaryDisplay = screen.getPrimaryDisplay();
723
- const sources = await desktopCapturer.getSources({
724
- types: ['screen'],
725
- thumbnailSize: {
726
- width: primaryDisplay.size.width,
727
- height: primaryDisplay.size.height
728
- }
729
- });
730
-
731
- if (sources.length > 0) {
732
- const screenImage = sources[0].thumbnail;
733
-
734
- // Crop the specific rect out of the full screen image
735
- // We use standard scale (assuming 1:1 scale for exact rect, although hidpi might vary.
736
- // In a robust implementation we might need `Math.floor(rect.x * display.scaleFactor)`)
737
- const croppedImage = screenImage.crop({
738
- x: Math.round(rect.x),
739
- y: Math.round(rect.y),
740
- width: Math.round(rect.width),
741
- height: Math.round(rect.height)
742
- });
743
-
744
- // Convert to JPEG Base64 for faster processing
745
- const base64Crop = croppedImage.toJPEG(70).toString('base64');
746
- const dataUri = `data:image/jpeg;base64,${base64Crop}`;
747
-
748
- const translationResult = await translateImageContent(dataUri);
749
- if (translationResult.retryableFailure) {
750
- translateCooldownUntil = Date.now() + TRANSLATE_FAILURE_COOLDOWN_MS;
751
- console.warn(`Live translation cooldown active for ${TRANSLATE_FAILURE_COOLDOWN_MS / 1000}s after retryable API failure.`);
752
- } else {
753
- translateCooldownUntil = 0;
754
- }
755
-
756
- if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
757
- screenPickerWindow.webContents.send('vision-translate-result', translationResult.text);
758
- }
759
- }
760
- } catch (err) {
761
- console.error("Continuous translation loop failed:", err);
762
- } finally {
763
- isTranslateRequestInFlight = false;
764
- }
765
- }
766
-
767
- ipcMain.on('vision-cancel', () => {
768
- if (translateIntervalHandle) {
769
- clearInterval(translateIntervalHandle);
770
- translateIntervalHandle = null;
771
- }
772
- isTranslateRequestInFlight = false;
773
- translateCooldownUntil = 0;
774
- if (screenPickerWindow) screenPickerWindow.close();
775
- if (mainWindow) mainWindow.show();
776
- });
777
-
778
- // =====================
779
- // IPC Handlers — Proactive Assistant
780
- // =====================
781
- ipcMain.on('toggle-proactive', (event, isOn) => {
782
- if (isOn) {
783
- startProactiveLoop(); // reads interval from config automatically
784
- } else {
785
- stopProactiveLoop();
786
- }
787
- });
788
-
789
- ipcMain.on('record-behavior', (event, contextDescription) => {
790
- recordBehavior(contextDescription);
791
- });
792
-
793
- // Direct action executor for proactive Accept (no Gemini involved)
794
- ipcMain.handle('execute-proactive-action', async (event, action) => {
795
- if (!action || action.type === 'none') {
796
- return { success: false, message: 'ไม่มี action ที่จะดำเนินการค่ะ' };
797
- }
798
- try {
799
- const result = await executeAction(action);
800
- // Build a friendly confirmation message
801
- const messages = {
802
- open_url: `เปิดเว็บไซต์ให้แล้วค่ะ 🌐`,
803
- open_app: `เปิดแอป ${action.target} ให้แล้วค่ะ 🚀`,
804
- search: `ค้นหา "${action.target}" ให้แล้วค่ะ 🔍`,
805
- web_automation: result || 'ดำเนินการเสร็จแล้วค่ะ ✅',
806
- create_folder: `สร้างโฟลเดอร์ "${action.target}" แล้วค่ะ 📁`,
807
- clipboard_write: `คัดลอกข้อความแล้วค่ะ 📋`,
808
- learn_file: result || `เรียนรู้เอกสารเรียบร้อยค่ะ 📚`,
809
- };
810
- return {
811
- success: true,
812
- message: messages[action.type] || 'ดำเนินการเสร็จแล้วค่ะ ✅'
813
- };
814
- } catch (err) {
815
- console.error('[ProactiveAction] Error:', err);
816
- return { success: false, message: `เกิดข้อผิดพลาด: ${err.message}` };
817
- }
818
- });
819
-
820
- ipcMain.handle('capture-silent-screen', async () => {
821
- try {
822
- const primaryDisplay = screen.getPrimaryDisplay();
823
- const { width, height } = primaryDisplay.size;
824
-
825
- const sources = await desktopCapturer.getSources({
826
- types: ['screen'],
827
- thumbnailSize: { width, height }
828
- });
829
-
830
- const primarySource = sources[0];
831
- if (primarySource && primarySource.thumbnail) {
832
- return primarySource.thumbnail.toDataURL();
833
- }
834
- return null;
835
- } catch (err) {
836
- console.error("Error silently capturing screen:", err);
837
- return null;
838
- }
839
- });
840
-
841
- // =====================
842
- // Action Executor
843
- // =====================
844
- async function executeAction(action) {
845
- console.log("Executing action:", action);
846
-
847
- switch (action.type) {
848
- case 'open_url':
849
- openWebsite(action.target);
850
- break;
851
- case 'search':
852
- openSearch(action.target);
853
- break;
854
- case 'open_app':
855
- openApp(action.target);
856
- break;
857
- case 'web_automation':
858
- return await performWebAutomation(action.target);
859
- case 'create_folder':
860
- createFolder(action.target);
861
- break;
862
- case 'open_file':
863
- const fileRes = await openFile(action.target);
864
- return fileRes || `Successfully opened file: ${action.target} ✅`;
865
- case 'open_folder':
866
- const folderRes = await openFile(action.target);
867
- return folderRes || `Successfully opened folder: ${action.target} ✅`;
868
- case 'delete_file':
869
- await deleteFile(action.target);
870
- break;
871
- case 'find_path': {
872
- const result = findPath(action.target, {
873
- type: action.pathType,
874
- maxResults: 10
875
- });
876
- if (!result.success) {
877
- return result.message;
878
- }
879
-
880
- if (action.openAfter === true) {
881
- if (result.matches.length === 1) {
882
- const match = result.matches[0];
883
- const openResult = await openFile(match.path);
884
- return openResult || `Successfully found and opened ${match.type === 'dir' ? 'folder' : 'file'}: ${match.path} ✅`;
885
- }
886
- return `Found multiple matches for "${action.target}". Please be more specific:\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
887
- }
888
-
889
- return `Found matches for "${action.target}":\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
890
- }
891
- case 'clipboard_write':
892
- clipboard.writeText(action.target);
893
- break;
894
- case 'learn_file':
895
- return await indexFile(action.target);
896
- case 'learn_folder':
897
- const { indexFolder } = require('./src/AI_Brain/knowledge_base');
898
- return await indexFolder(action.target);
899
- case 'mcp_tool':
900
- const mcpResult = await mcpManager.callTool(action.server, action.target, action.args);
901
- return JSON.stringify(mcpResult.content);
902
- case 'mouse_move':
903
- return await granularAutomation.mouseMove(action.x, action.y);
904
- case 'mouse_click':
905
- return await granularAutomation.mouseClick(action.x, action.y, action.button || 1);
906
- case 'type_text':
907
- return await granularAutomation.typeText(action.target);
908
- case 'key_tap':
909
- return await granularAutomation.keyTap(action.target);
910
- case 'plugin':
911
- return await pluginManager.executePlugin(action.pluginName, action.target);
912
- case 'system_automation':
913
- return await handleSystemAutomation(action.target);
914
- }
915
- }
916
-
917
- async function handleSystemAutomation(target) {
918
- const [cmd, value] = target.split(':');
919
- switch (cmd) {
920
- case 'volume':
921
- return await SystemAutomation.setVolume(parseInt(value));
922
- case 'mute':
923
- return await SystemAutomation.mute();
924
- case 'brightness':
925
- return await SystemAutomation.setBrightness(parseInt(value));
926
- case 'sleep':
927
- return await SystemAutomation.sleep();
928
- case 'restart':
929
- return await SystemAutomation.restart();
930
- case 'shutdown':
931
- return await SystemAutomation.shutdown();
932
- case 'minimize_all':
933
- return await SystemAutomation.minimizeAll();
934
- default:
935
- // Handle cases where target is just the command (like 'sleep')
936
- if (SystemAutomation[target]) {
937
- return await SystemAutomation[target]();
938
- }
939
- throw new Error(`Unknown system automation command: ${target}`);
940
- }
941
- }