@pheem49/mint 1.4.2 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/GUIDE_TH.md +113 -0
  2. package/README.md +267 -78
  3. package/assets/CLI_Screen.png +0 -0
  4. package/main.js +76 -890
  5. package/mint-cli-logic.js +3 -107
  6. package/mint-cli.js +594 -29
  7. package/models/Shiroko_Model/Shiroko/Shiroko_Core/72d86db84cfa9730b894c241fd24c0db.png +0 -0
  8. package/models/Shiroko_Model/Shiroko/Shiroko_Core/items_pinned_to_model.json +14 -0
  9. package/models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253.exp3.json +10 -0
  10. package/models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253/347/234/274/347/217/240/346/221/207/346/231/203.exp3.json +15 -0
  11. package/models/Shiroko_Model/Shiroko/Shiroko_Core//345/233/264/350/243/231.exp3.json +10 -0
  12. package/models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/215/347/205/247.exp3.json +50 -0
  13. package/models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/277/347/254/224.exp3.json +10 -0
  14. package/models/Shiroko_Model/Shiroko/Shiroko_Core//347/202/271/344/270/200/344/270/213.exp3.json +10 -0
  15. package/models/Shiroko_Model/Shiroko/Shiroko_Core//347/214/253/345/222/252/346/273/244/351/225/234.exp3.json +10 -0
  16. package/models/Shiroko_Model/Shiroko/Shiroko_Core//347/234/274/351/225/234.exp3.json +10 -0
  17. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_00.png +0 -0
  18. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_01.png +0 -0
  19. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_02.png +0 -0
  20. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_03.png +0 -0
  21. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.cdi3.json +1498 -0
  22. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.moc3 +0 -0
  23. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.model3.json +47 -0
  24. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.physics3.json +6658 -0
  25. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.vtube.json +1299 -0
  26. package/models/Shiroko_Model/Shiroko//342/232/241/351/253/230/344/272/256/342/232/241/344/275/277/347/224/250/346/225/231/347/250/213/344/270/216/346/263/250/346/204/217/344/272/213/351/241/271.txt +23 -0
  27. package/package.json +37 -4
  28. package/src/AI_Brain/Gemini_API.js +223 -65
  29. package/src/AI_Brain/autonomous_brain.js +11 -0
  30. package/src/AI_Brain/behavior_memory.js +26 -5
  31. package/src/AI_Brain/headless_agent.js +4 -0
  32. package/src/AI_Brain/knowledge_base.js +61 -8
  33. package/src/AI_Brain/memory_store.js +354 -10
  34. package/src/Automation_Layer/file_operations.js +1 -1
  35. package/src/CLI/chat_router.js +20 -7
  36. package/src/CLI/chat_ui.js +596 -825
  37. package/src/CLI/code_agent.js +347 -56
  38. package/src/CLI/gmail_auth.js +210 -0
  39. package/src/CLI/image_input.js +90 -0
  40. package/src/CLI/list_features.js +2 -0
  41. package/src/CLI/onboarding.js +364 -55
  42. package/src/CLI/updater.js +210 -0
  43. package/src/Channels/brave_search_bridge.js +35 -0
  44. package/src/Channels/discord_bridge.js +68 -0
  45. package/src/Channels/google_search_bridge.js +38 -0
  46. package/src/Channels/line_bridge.js +60 -0
  47. package/src/Channels/slack_bridge.js +53 -0
  48. package/src/Channels/telegram_bridge.js +49 -0
  49. package/src/Channels/whatsapp_bridge.js +55 -0
  50. package/src/Command_Parser/parser.js +12 -1
  51. package/src/Plugins/gmail.js +251 -0
  52. package/src/Plugins/google_calendar.js +245 -19
  53. package/src/Plugins/notion.js +256 -0
  54. package/src/System/action_executor.js +178 -0
  55. package/src/System/bridge_manager.js +76 -0
  56. package/src/System/chat_history_manager.js +23 -5
  57. package/src/System/config_manager.js +71 -7
  58. package/src/System/custom_workflows.js +31 -2
  59. package/src/System/google_tts_urls.js +51 -0
  60. package/src/System/granular_automation.js +122 -53
  61. package/src/System/ipc_handlers.js +238 -0
  62. package/src/System/proactive_loop.js +153 -0
  63. package/src/System/safety_manager.js +273 -0
  64. package/src/System/sandbox_runner.js +182 -0
  65. package/src/System/screen_capture.js +175 -0
  66. package/src/System/system_automation.js +127 -81
  67. package/src/System/system_info.js +70 -0
  68. package/src/System/task_manager.js +15 -5
  69. package/src/System/tool_registry.js +280 -0
  70. package/src/System/window_manager.js +212 -0
  71. package/src/UI/live2d_manager.js +368 -0
  72. package/src/UI/renderer.js +208 -24
  73. package/src/UI/settings.html +24 -0
  74. package/src/UI/settings.js +14 -4
  75. package/src/UI/styles.css +466 -32
  76. package/.codex +0 -0
  77. package/docs/assets/Agent_Mint.png +0 -0
  78. package/docs/assets/CLI_Screen.png +0 -0
  79. package/docs/assets/Settings.png +0 -0
  80. package/docs/assets/icon.png +0 -0
  81. package/docs/index.html +0 -132
  82. package/docs/style.css +0 -579
  83. package/index.html +0 -16
  84. package/src/UI/index.html +0 -126
  85. package/tech_news.txt +0 -3
  86. package/test_knowledge.txt +0 -3
  87. package/tests/agent_orchestrator.test.js +0 -41
  88. package/tests/chat_router.test.js +0 -42
  89. package/tests/code_agent.test.js +0 -69
  90. package/tests/config_manager.test.js +0 -141
  91. package/tests/docker.test.js +0 -46
  92. package/tests/file_operations.test.js +0 -57
  93. package/tests/memory_store.test.js +0 -185
  94. package/tests/provider_routing.test.js +0 -67
  95. package/tests/spotify.test.js +0 -201
  96. package/tests/system_monitor.test.js +0 -37
  97. package/tests/workspace_manager.test.js +0 -56
package/main.js CHANGED
@@ -1,360 +1,96 @@
1
- const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard, Tray, Menu, nativeImage, desktopCapturer, screen, powerMonitor } = require('electron');
2
- const path = require('path');
1
+ const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard } = require('electron');
3
2
  require('dotenv').config();
4
3
 
5
- const { handleChat, resetChat, getChatTranscript, translateImageContent, refreshApiKeyFromConfig } = require('./src/AI_Brain/Gemini_API');
6
- const { openApp } = require('./src/Automation_Layer/open_app');
7
- const { openWebsite, openSearch } = require('./src/Automation_Layer/open_website');
8
- const { performWebAutomation } = require('./src/Automation_Layer/browser_automation');
9
- const { createFolder, openFile, deleteFile, findPath } = require('./src/Automation_Layer/file_operations');
10
4
  const { getSystemInfo, getWeather } = require('./src/System/system_info');
11
5
  const { readConfig, writeConfig } = require('./src/System/config_manager');
12
6
  const { parseCommand } = require('./src/Command_Parser/parser');
13
- const pluginManager = require('./src/Plugins/plugin_manager');
14
- const { analyzeAndSuggest } = require('./src/AI_Brain/proactive_engine');
15
- const { recordBehavior, getBehaviorSummary } = require('./src/AI_Brain/behavior_memory');
16
- const { indexFile, indexFolder } = require('./src/AI_Brain/knowledge_base');
7
+ const { getGoogleTtsUrls } = require('./src/System/google_tts_urls');
8
+ const { createWindowManager } = require('./src/System/window_manager');
9
+ const { createProactiveLoop } = require('./src/System/proactive_loop');
10
+ const { createScreenCaptureController } = require('./src/System/screen_capture');
11
+ const { registerIpcHandlers } = require('./src/System/ipc_handlers');
17
12
 
18
- const SystemAutomation = require('./src/System/system_automation');
19
13
  const systemEvents = require('./src/System/system_events');
20
14
  const customWorkflows = require('./src/System/custom_workflows');
21
15
  const mcpManager = require('./src/Plugins/mcp_manager');
22
- const granularAutomation = require('./src/System/granular_automation');
23
- const googleTTS = require('google-tts-api');
24
16
 
25
- let mainWindow;
26
- let settingsWindow = null;
27
- let screenPickerWindow = null;
28
- let spotlightWindow = null;
29
- let widgetWindow = null;
30
- let proactiveGlowWindow = null;
31
- let tray = null;
32
-
33
- // =====================
34
- // Proactive Loop
35
- // =====================
36
- let proactiveIntervalHandle = null;
37
-
38
- async function runProactiveCycle() {
39
- if (!mainWindow || mainWindow.isDestroyed()) return;
40
- try {
41
- // Show Glow
42
- showProactiveGlow();
43
-
44
- // Silent screen capture
45
- const primaryDisplay = screen.getPrimaryDisplay();
46
- // Downscale to 50% for performance
47
- const width = Math.floor(primaryDisplay.size.width * 0.5);
48
- const height = Math.floor(primaryDisplay.size.height * 0.5);
49
-
50
- const sources = await desktopCapturer.getSources({
51
- types: ['screen'],
52
- thumbnailSize: { width, height }
53
- });
54
- const primarySource = sources[0];
55
- if (!primarySource || !primarySource.thumbnail) return;
56
-
57
- // Compress image to JPEG to save huge base64 payload size and memory
58
- // .toJPEG(quality) where quality is 0-100
59
- const base64Image = primarySource.thumbnail.toJPEG(60).toString('base64');
60
- const behaviorSummary = getBehaviorSummary();
61
-
62
- const result = await analyzeAndSuggest(base64Image, behaviorSummary);
63
-
64
- if (result && result.message && Array.isArray(result.suggestions)) {
65
- // Record the observed context into behavior memory
66
- if (result.context) recordBehavior(result.context);
67
-
68
- // Push suggestion to the renderer
69
- if (mainWindow && !mainWindow.isDestroyed()) {
70
- mainWindow.webContents.send('proactive-suggestion', result);
71
- }
72
- }
73
-
74
- // Hide Glow after analysis
75
- hideProactiveGlow();
76
- } catch (err) {
77
- console.error('[Proactive] Cycle error:', err.message);
78
- hideProactiveGlow();
79
- }
80
- }
81
-
82
- function createProactiveGlowWindow() {
83
- if (proactiveGlowWindow) return;
84
- const { width, height } = screen.getPrimaryDisplay().bounds;
85
-
86
- proactiveGlowWindow = new BrowserWindow({
87
- width, height,
88
- x: 0, y: 0,
89
- frame: false,
90
- transparent: true,
91
- alwaysOnTop: true,
92
- skipTaskbar: true,
93
- focusable: false,
94
- show: false,
95
- webPreferences: {
96
- nodeIntegration: false,
97
- contextIsolation: true
98
- }
99
- });
100
-
101
- proactiveGlowWindow.setIgnoreMouseEvents(true);
102
- proactiveGlowWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
103
- proactiveGlowWindow.setAlwaysOnTop(true, 'screen-saver');
104
- proactiveGlowWindow.loadFile('src/UI/proactive-glow.html');
105
-
106
- proactiveGlowWindow.on('closed', () => { proactiveGlowWindow = null; });
107
- }
108
-
109
- function showProactiveGlow() {
110
- if (!proactiveGlowWindow) createProactiveGlowWindow();
111
- if (proactiveGlowWindow) proactiveGlowWindow.showInactive();
112
- }
113
-
114
- function hideProactiveGlow() {
115
- if (proactiveGlowWindow && !proactiveGlowWindow.isDestroyed()) {
116
- proactiveGlowWindow.hide();
117
- }
118
- }
119
-
120
- function startProactiveLoop(intervalSec) {
121
- // Stop existing loop first
122
- if (proactiveIntervalHandle) {
123
- clearInterval(proactiveIntervalHandle);
124
- proactiveIntervalHandle = null;
125
- }
126
- // Read from config if not provided
127
- const cfg = readConfig();
128
- const ms = (intervalSec || cfg.proactiveInterval || 60) * 1000;
129
- console.log(`[Proactive] Starting loop — interval: ${ms / 1000}s`);
130
- proactiveIntervalHandle = setInterval(runProactiveCycle, ms);
131
- }
132
-
133
- function stopProactiveLoop() {
134
- if (proactiveIntervalHandle) {
135
- clearInterval(proactiveIntervalHandle);
136
- proactiveIntervalHandle = null;
137
- console.log('[Proactive] Stopped proactive loop.');
138
- }
139
- }
140
-
141
- // Check idle state every minute to pause/resume background tasks
142
- const IDLE_THRESHOLD_SEC = 300; // 5 minutes
143
- setInterval(() => {
144
- if (!proactiveIntervalHandle) return; // Only manage if it's supposed to be active (Smart Context is ON)
145
-
146
- // powerMonitor.getSystemIdleTime() is available after app is ready
147
- if (app.isReady()) {
148
- const idleSec = powerMonitor.getSystemIdleTime();
149
- if (idleSec >= IDLE_THRESHOLD_SEC) {
150
- console.log(`[System Idle] User idle for ${idleSec}s. Pausing Proactive loop to save resources.`);
151
- stopProactiveLoop();
152
-
153
- // Wait for user to come back
154
- const resumeChecker = setInterval(() => {
155
- if (powerMonitor.getSystemIdleTime() < 10) {
156
- console.log('[System Idle] User returned. Resuming Proactive loop.');
157
- clearInterval(resumeChecker);
158
- startProactiveLoop();
159
- }
160
- }, 5000);
161
- }
17
+ let geminiServices = null;
18
+ function getGeminiServices() {
19
+ if (!geminiServices) {
20
+ geminiServices = require('./src/AI_Brain/Gemini_API');
162
21
  }
163
- }, 60000);
164
-
165
- function createWindow() {
166
- mainWindow = new BrowserWindow({
167
- width: 600,
168
- height: 800,
169
- icon: path.join(__dirname, 'assets', 'icon.png'),
170
- webPreferences: {
171
- preload: path.join(__dirname, 'preload.js'),
172
- nodeIntegration: false,
173
- contextIsolation: true,
174
- },
175
- frame: false,
176
- transparent: true,
177
- resizable: true,
178
- show: false
179
- });
180
-
181
- mainWindow.loadFile('src/UI/index.html');
182
-
183
- mainWindow.on('ready-to-show', () => {
184
- mainWindow.show();
185
- });
186
-
187
- mainWindow.on('close', function (event) {
188
- if (!app.isQuiting) {
189
- event.preventDefault();
190
- mainWindow.hide();
191
- }
192
- return false;
193
- });
194
-
195
- mainWindow.on('focus', () => {
196
- // clearFloatingUnread(); // Disabled
197
- });
22
+ return geminiServices;
198
23
  }
199
24
 
200
- function createTray() {
201
- const iconPath = path.join(__dirname, 'assets', 'icon.png');
202
- let icon = nativeImage.createFromPath(iconPath);
203
- // Optional: resize it for the tray if needed, but Electron handles this natively on many OSs
204
- icon = icon.resize({ width: 16, height: 16 });
205
-
206
- tray = new Tray(icon);
207
- tray.setToolTip('Mint AI Assistant');
208
-
209
- const contextMenu = Menu.buildFromTemplate([
210
- { label: 'Show App', click: () => { if (mainWindow) mainWindow.show(); } },
211
- { label: 'Settings', click: () => { createSettingsWindow(); } },
212
- { type: 'separator' },
213
- { label: 'Quit', click: () => {
214
- app.isQuiting = true;
215
- app.quit();
216
- }}
217
- ]);
218
-
219
- tray.setContextMenu(contextMenu);
220
-
221
- tray.on('click', () => {
222
- if (mainWindow) {
223
- if (mainWindow.isVisible()) {
224
- mainWindow.hide();
225
- } else {
226
- mainWindow.show();
227
- }
228
- }
229
- });
230
- }
231
-
232
- function createSettingsWindow() {
233
- if (settingsWindow) {
234
- settingsWindow.focus();
235
- return;
236
- }
237
- settingsWindow = new BrowserWindow({
238
- width: 720,
239
- height: 620,
240
- minWidth: 640,
241
- minHeight: 560,
242
- icon: path.join(__dirname, 'assets', 'icon.png'),
243
- webPreferences: {
244
- preload: path.join(__dirname, 'preload-settings.js'),
245
- nodeIntegration: false,
246
- contextIsolation: true,
247
- },
248
- frame: false,
249
- transparent: true,
250
- resizable: true,
251
- parent: mainWindow,
252
- });
253
- settingsWindow.loadFile('src/UI/settings.html');
254
- settingsWindow.on('closed', () => { settingsWindow = null; });
25
+ function getActionExecutor() {
26
+ return require('./src/System/action_executor');
255
27
  }
256
28
 
257
- function createSpotlightWindow() {
258
- if (spotlightWindow) {
259
- spotlightWindow.show();
260
- return;
29
+ const projectRoot = __dirname;
30
+ const windowManager = createWindowManager(projectRoot);
31
+ const proactiveLoop = createProactiveLoop({
32
+ app,
33
+ projectRoot,
34
+ readConfig,
35
+ getMainWindow: windowManager.getMainWindow
36
+ });
37
+ const screenCapture = createScreenCaptureController({
38
+ projectRoot,
39
+ translateImageContent: (...args) => getGeminiServices().translateImageContent(...args),
40
+ getMainWindow: windowManager.getMainWindow
41
+ });
42
+
43
+ registerIpcHandlers({
44
+ app,
45
+ ipcMain,
46
+ shell,
47
+ clipboard,
48
+ windowManager,
49
+ proactiveLoop,
50
+ screenCapture,
51
+ services: {
52
+ handleChat: (...args) => getGeminiServices().handleChat(...args),
53
+ resetChat: (...args) => getGeminiServices().resetChat(...args),
54
+ getChatTranscript: (...args) => getGeminiServices().getChatTranscript(...args),
55
+ refreshApiKeyFromConfig: (...args) => getGeminiServices().refreshApiKeyFromConfig(...args),
56
+ getSystemInfo,
57
+ getWeather,
58
+ readConfig,
59
+ writeConfig,
60
+ parseCommand,
61
+ executeAction: (...args) => getActionExecutor().executeAction(...args),
62
+ getGoogleTtsUrls,
63
+ customWorkflows
261
64
  }
65
+ });
262
66
 
263
- const { width: screenWidth, height: screenHeight } = screen.getPrimaryDisplay().workAreaSize;
264
- const windowWidth = 600;
265
- const windowHeight = 80; // Starts small, expands if results show
266
-
267
- spotlightWindow = new BrowserWindow({
268
- width: windowWidth,
269
- height: windowHeight,
270
- x: Math.floor((screenWidth - windowWidth) / 2),
271
- y: Math.floor(screenHeight * 0.25), // 25% from top
272
- frame: false,
273
- transparent: true,
274
- alwaysOnTop: true,
275
- skipTaskbar: true,
276
- show: false,
277
- webPreferences: {
278
- preload: path.join(__dirname, 'src/UI/preload-spotlight.js'),
279
- nodeIntegration: false,
280
- contextIsolation: true,
281
- }
282
- });
283
-
284
- spotlightWindow.loadFile('src/UI/spotlight.html');
285
-
286
- spotlightWindow.on('blur', () => {
287
- spotlightWindow.hide();
288
- });
289
-
290
- spotlightWindow.on('closed', () => {
291
- spotlightWindow = null;
292
- });
293
- }
294
-
295
- // Floating window logic removed
67
+ app.whenReady().then(() => {
68
+ const config = readConfig();
69
+ const mainWindow = windowManager.createMainWindow();
70
+ windowManager.createTray();
296
71
 
297
- function createWidgetWindow() {
298
- if (widgetWindow) return;
299
-
300
- widgetWindow = new BrowserWindow({
301
- width: 150,
302
- height: 150,
303
- frame: false,
304
- transparent: true,
305
- resizable: false,
306
- alwaysOnTop: true,
307
- skipTaskbar: true,
308
- show: true,
309
- webPreferences: {
310
- preload: require('path').join(__dirname, 'src/UI/preload-widget.js'),
311
- nodeIntegration: false,
312
- contextIsolation: true
72
+ mainWindow.once('ready-to-show', () => {
73
+ if (config.showDesktopWidget !== false) {
74
+ setTimeout(() => windowManager.createWidgetWindow(), 300);
313
75
  }
314
- });
315
-
316
- widgetWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
317
- widgetWindow.setAlwaysOnTop(true, 'floating');
318
-
319
- // Position Top-Right loosely
320
- try {
321
- const primaryDisplay = require('electron').screen.getPrimaryDisplay();
322
- const { width, height, x, y } = primaryDisplay.workArea;
323
- widgetWindow.setPosition(x + width - 150 - 40, y + 40);
324
- } catch (e) {}
325
76
 
326
- widgetWindow.loadFile('src/UI/widget.html');
77
+ setTimeout(() => {
78
+ mcpManager.init().catch(err => console.error('[MCP] Init Error:', err));
327
79
 
328
- widgetWindow.on('closed', () => {
329
- widgetWindow = null;
80
+ const bridgeManager = require('./src/System/bridge_manager');
81
+ bridgeManager.init().catch(err => console.error('[BridgeManager] Init Error:', err));
82
+ }, 1000);
330
83
  });
331
- }
332
84
 
333
- // Floating unread logic removed
334
-
335
- app.whenReady().then(() => {
336
- const config = readConfig();
337
- createWindow();
338
- createTray();
339
- // createFloatingWindow(); // Removed
340
-
341
- // Only show AI widget if enabled in settings
342
- if (config.showDesktopWidget !== false) {
343
- createWidgetWindow();
344
- }
345
-
346
- // Initialize MCP Servers
347
- mcpManager.init().catch(err => console.error('[MCP] Init Error:', err));
348
-
349
- // Start monitoring system events (battery, wifi, etc.)
350
85
  systemEvents.startMonitoring();
351
86
  if (config.enableCustomWorkflows !== false) {
352
87
  customWorkflows.startMonitoring(mainWindow.webContents);
353
88
  }
354
89
 
355
90
  systemEvents.on('low-battery', (level) => {
356
- if (mainWindow && !mainWindow.isDestroyed()) {
357
- mainWindow.webContents.send('proactive-notification', {
91
+ const currentMainWindow = windowManager.getMainWindow();
92
+ if (currentMainWindow && !currentMainWindow.isDestroyed()) {
93
+ currentMainWindow.webContents.send('proactive-notification', {
358
94
  message: `⚠️ Battery is low (${level}%). Please plug in your charger. ✨`,
359
95
  type: 'warning'
360
96
  });
@@ -362,580 +98,30 @@ app.whenReady().then(() => {
362
98
  });
363
99
 
364
100
  systemEvents.on('connection-change', (isOnline) => {
365
- if (mainWindow && !mainWindow.isDestroyed()) {
101
+ const currentMainWindow = windowManager.getMainWindow();
102
+ if (currentMainWindow && !currentMainWindow.isDestroyed()) {
366
103
  const msg = isOnline ? '✅ Internet connection restored. ✨' : '❌ Internet connection lost.';
367
- mainWindow.webContents.send('proactive-notification', { message: msg, type: 'info' });
104
+ currentMainWindow.webContents.send('proactive-notification', { message: msg, type: 'info' });
368
105
  }
369
106
  });
370
107
 
371
- globalShortcut.register('CommandOrControl+Shift+Space', () => {
372
- if (mainWindow) {
373
- if (mainWindow.isVisible()) {
374
- mainWindow.hide();
375
- } else {
376
- mainWindow.show();
377
- }
378
- }
379
- });
108
+ globalShortcut.register('CommandOrControl+Shift+Space', windowManager.toggleMainWindow);
109
+ globalShortcut.register('Alt+Space', windowManager.toggleSpotlightWindow);
380
110
 
381
- globalShortcut.register('Alt+Space', () => {
382
- if (spotlightWindow && spotlightWindow.isVisible()) {
383
- spotlightWindow.hide();
384
- } else {
385
- createSpotlightWindow();
386
- spotlightWindow.show();
387
- }
388
- });
111
+ proactiveLoop.startIdleWatcher();
389
112
 
390
- app.on('activate', function () {
391
- if (BrowserWindow.getAllWindows().length === 0) createWindow();
113
+ app.on('activate', () => {
114
+ if (BrowserWindow.getAllWindows().length === 0) {
115
+ windowManager.createMainWindow();
116
+ }
392
117
  });
393
118
  });
394
119
 
395
- app.on('window-all-closed', (e) => {
396
- e.preventDefault();
120
+ app.on('window-all-closed', (event) => {
121
+ event.preventDefault();
397
122
  });
398
123
 
399
124
  app.on('will-quit', () => {
400
125
  globalShortcut.unregisterAll();
401
126
  mcpManager.shutdown();
402
127
  });
403
-
404
- // =====================
405
- // IPC Handlers — Chat
406
- // =====================
407
- ipcMain.handle('chat-message', async (event, message, base64Image = null, base64Audio = null) => {
408
- try {
409
- const rawResponse = await handleChat(message, base64Image, base64Audio);
410
- const aiResponse = parseCommand(rawResponse);
411
-
412
- if (aiResponse.action && aiResponse.action.type !== 'none') {
413
- try {
414
- const actionResult = await executeAction(aiResponse.action);
415
- // If the action returned a string result (e.g. from Web Automation), append it.
416
- if (actionResult && typeof actionResult === 'string') {
417
- aiResponse.response += `\n\n${actionResult}`;
418
- }
419
- } catch (err) {
420
- console.error("Action execution error:", err);
421
- aiResponse.response += "\n\n(Note: I tried to execute the action, but an error occurred.)";
422
- }
423
- }
424
-
425
- return aiResponse;
426
- } catch (error) {
427
- console.error('Chat error:', error);
428
- return { response: 'Error communicating with Gemini API. Check your console and API key.', action: { type: 'none' } };
429
- }
430
- });
431
-
432
- ipcMain.on('close-window', (event) => {
433
- if (mainWindow) mainWindow.hide();
434
- });
435
-
436
- ipcMain.on('minimize-window', (event) => {
437
- if (mainWindow) mainWindow.minimize();
438
- });
439
-
440
- ipcMain.on('quit-app', (event) => {
441
- app.isQuiting = true;
442
- app.quit();
443
- });
444
-
445
- ipcMain.on('maximize-window', (event) => {
446
- if (mainWindow) {
447
- if (mainWindow.isMaximized()) {
448
- mainWindow.unmaximize();
449
- } else {
450
- mainWindow.maximize();
451
- }
452
- }
453
- });
454
-
455
- ipcMain.handle('reset-chat', () => {
456
- resetChat();
457
- return { success: true };
458
- });
459
-
460
- ipcMain.handle('get-chat-history', () => {
461
- return getChatTranscript();
462
- });
463
-
464
- // =====================
465
- // IPC Handlers — Settings
466
- // =====================
467
- ipcMain.handle('open-settings', () => {
468
- createSettingsWindow();
469
- });
470
-
471
- ipcMain.handle('get-settings', () => {
472
- return readConfig();
473
- });
474
-
475
- ipcMain.handle('save-settings', (event, config) => {
476
- console.log('[Settings] Saving new config. MCP Servers count:', Object.keys(config.mcpServers || {}).length);
477
- const result = writeConfig(config);
478
- // Refresh API key if user updated it in settings
479
- refreshApiKeyFromConfig();
480
- // 🔔 Notify main chat window immediately
481
- if (mainWindow && !mainWindow.isDestroyed()) {
482
- mainWindow.webContents.send('settings-changed', config);
483
- }
484
- // Restart proactive loop with new interval if it's running
485
- if (proactiveIntervalHandle) {
486
- startProactiveLoop(config.proactiveInterval);
487
- }
488
- if (config.enableCustomWorkflows !== false) {
489
- customWorkflows.startMonitoring(mainWindow.webContents);
490
- } else {
491
- customWorkflows.stopMonitoring();
492
- }
493
-
494
- // Toggle Desktop Widget
495
- if (config.showDesktopWidget === false) {
496
- if (widgetWindow && !widgetWindow.isDestroyed()) {
497
- widgetWindow.close();
498
- widgetWindow = null;
499
- }
500
- } else {
501
- if (!widgetWindow || widgetWindow.isDestroyed()) {
502
- createWidgetWindow();
503
- }
504
- }
505
-
506
- return result;
507
- });
508
-
509
- ipcMain.on('set-ai-state', (event, state) => {
510
- if (widgetWindow && !widgetWindow.isDestroyed()) {
511
- widgetWindow.webContents.send('widget-state', state);
512
- }
513
- });
514
-
515
- ipcMain.on('close-settings', () => {
516
- if (settingsWindow) settingsWindow.close();
517
- });
518
-
519
- ipcMain.on('quit-app', () => {
520
- app.isQuiting = true;
521
- app.quit();
522
- });
523
-
524
- ipcMain.handle('open-custom-workflows', () => {
525
- customWorkflows.openConfigFile();
526
- });
527
-
528
- ipcMain.handle('reload-custom-workflows', () => {
529
- customWorkflows.loadWorkflows();
530
- return { success: true };
531
- });
532
-
533
- // =====================
534
- // IPC Handlers — Spotlight
535
- // =====================
536
- ipcMain.on('spotlight-close', () => {
537
- if (spotlightWindow) spotlightWindow.close();
538
- });
539
-
540
- ipcMain.on('spotlight-hide', () => {
541
- if (spotlightWindow) spotlightWindow.hide();
542
- });
543
-
544
- ipcMain.on('spotlight-submit', async (event, query) => {
545
- console.log('[Spotlight] Submit:', query);
546
- if (spotlightWindow) spotlightWindow.hide();
547
-
548
- // Show main window and send the message
549
- if (mainWindow) {
550
- mainWindow.show();
551
- // We can either just show it or actually trigger the chat
552
- mainWindow.webContents.send('spotlight-to-chat', query);
553
- }
554
- });
555
-
556
- ipcMain.on('spotlight-resize', (event, width, height) => {
557
- if (spotlightWindow) {
558
- spotlightWindow.setSize(width, height);
559
- }
560
- });
561
-
562
- // Floating IPC removed
563
-
564
- ipcMain.handle('open-external', (event, url) => {
565
- shell.openExternal(url);
566
- });
567
-
568
- // =====================
569
- // IPC Handlers — Clipboard
570
- // =====================
571
- ipcMain.handle('clipboard-read', () => {
572
- return clipboard.readText();
573
- });
574
-
575
- ipcMain.handle('clipboard-write', (event, text) => {
576
- clipboard.writeText(text);
577
- return { success: true };
578
- });
579
-
580
- // =====================
581
- // IPC Handlers — TTS
582
- // =====================
583
- ipcMain.handle('get-tts-urls', async (event, text) => {
584
- try {
585
- const isThai = /[\u0E00-\u0E7F]/.test(text);
586
- const lang = isThai ? 'th' : 'en';
587
-
588
- const results = googleTTS.getAllAudioUrls(text, {
589
- lang: lang,
590
- slow: false,
591
- host: 'https://translate.google.com',
592
- splitPunct: ',.?!;:',
593
- });
594
- return results;
595
- } catch (e) {
596
- console.error("TTS Error:", e);
597
- return [];
598
- }
599
- });
600
-
601
- // =====================
602
- // IPC Handlers — System Info
603
- // =====================
604
- ipcMain.handle('get-system-info', async () => {
605
- return getSystemInfo();
606
- });
607
-
608
- ipcMain.handle('get-weather', async (event, city) => {
609
- return getWeather(city);
610
- });
611
-
612
- // =====================
613
- // IPC Handlers — Screen Vision
614
- // =====================
615
- ipcMain.handle('start-screen-capture', async () => {
616
- if (screenPickerWindow) return; // Prevent multiple windows
617
-
618
- try {
619
- // Capture primary display
620
- const primaryDisplay = screen.getPrimaryDisplay();
621
- const { width, height } = primaryDisplay.size;
622
-
623
- const sources = await desktopCapturer.getSources({
624
- types: ['screen'],
625
- thumbnailSize: { width, height }
626
- });
627
-
628
- // Find the full primary screen (usually "Screen 1" or "Entire screen")
629
- const primarySource = sources[0]; // Assuming single/primary monitor is the first
630
-
631
- // Create transparent, borderless screen picker window
632
- screenPickerWindow = new BrowserWindow({
633
- width, height,
634
- x: primaryDisplay.bounds.x, y: primaryDisplay.bounds.y,
635
- fullscreen: true,
636
- transparent: true,
637
- frame: false,
638
- alwaysOnTop: true,
639
- skipTaskbar: true,
640
- webPreferences: {
641
- preload: path.join(__dirname, 'preload-picker.js'),
642
- nodeIntegration: false,
643
- contextIsolation: true
644
- }
645
- });
646
-
647
- await screenPickerWindow.loadFile('src/UI/screenPicker.html');
648
-
649
- // Ensure image data isn't missing
650
- if(primarySource && primarySource.thumbnail) {
651
- screenPickerWindow.webContents.send('screenshot-data', primarySource.thumbnail.toDataURL());
652
- }
653
-
654
- screenPickerWindow.on('closed', () => { screenPickerWindow = null; });
655
- } catch (err) {
656
- console.error("Error starting screen capture:", err);
657
- }
658
- });
659
-
660
- // Received selection from the picker
661
- ipcMain.on('vision-selection', (event, base64Image) => {
662
- if (screenPickerWindow) screenPickerWindow.close();
663
-
664
- // Relay image back to main window interface
665
- if (mainWindow) {
666
- mainWindow.webContents.send('vision-ready', base64Image);
667
- mainWindow.show(); // Bring chat back to focus
668
- }
669
- });
670
-
671
- let translateIntervalHandle = null;
672
- let isTranslateRequestInFlight = false;
673
- let translateCooldownUntil = 0;
674
- const TRANSLATE_REFRESH_MS = 3000;
675
- const TRANSLATE_FAILURE_COOLDOWN_MS = 15000;
676
-
677
- // Start Continuous Live Translate
678
- ipcMain.on('vision-translate-start', async (event, rect) => {
679
- if (!screenPickerWindow) return;
680
-
681
- screenPickerWindow.setIgnoreMouseEvents(true, { forward: true });
682
- isTranslateRequestInFlight = false;
683
- translateCooldownUntil = 0;
684
-
685
- // Stop any existing loop
686
- if (translateIntervalHandle) clearInterval(translateIntervalHandle);
687
-
688
- // Initial capture immediately
689
- captureAndTranslate(rect);
690
-
691
- // Then start ticking every 3 seconds
692
- translateIntervalHandle = setInterval(() => {
693
- captureAndTranslate(rect);
694
- }, TRANSLATE_REFRESH_MS);
695
- });
696
-
697
- // Stop Continuous Live Translate
698
- ipcMain.on('vision-translate-stop', () => {
699
- if (translateIntervalHandle) {
700
- clearInterval(translateIntervalHandle);
701
- translateIntervalHandle = null;
702
- }
703
- if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
704
- screenPickerWindow.setIgnoreMouseEvents(false);
705
- }
706
- isTranslateRequestInFlight = false;
707
- translateCooldownUntil = 0;
708
- });
709
-
710
- ipcMain.on('vision-overlay-interactable', (event, isInteractable) => {
711
- if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
712
- screenPickerWindow.setIgnoreMouseEvents(!isInteractable, { forward: true });
713
- });
714
-
715
- async function captureAndTranslate(rect) {
716
- if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
717
- if (isTranslateRequestInFlight) return;
718
- if (Date.now() < translateCooldownUntil) return;
719
-
720
- try {
721
- isTranslateRequestInFlight = true;
722
- const primaryDisplay = screen.getPrimaryDisplay();
723
- const sources = await desktopCapturer.getSources({
724
- types: ['screen'],
725
- thumbnailSize: {
726
- width: primaryDisplay.size.width,
727
- height: primaryDisplay.size.height
728
- }
729
- });
730
-
731
- if (sources.length > 0) {
732
- const screenImage = sources[0].thumbnail;
733
-
734
- // Crop the specific rect out of the full screen image
735
- // We use standard scale (assuming 1:1 scale for exact rect, although hidpi might vary.
736
- // In a robust implementation we might need `Math.floor(rect.x * display.scaleFactor)`)
737
- const croppedImage = screenImage.crop({
738
- x: Math.round(rect.x),
739
- y: Math.round(rect.y),
740
- width: Math.round(rect.width),
741
- height: Math.round(rect.height)
742
- });
743
-
744
- // Convert to JPEG Base64 for faster processing
745
- const base64Crop = croppedImage.toJPEG(70).toString('base64');
746
- const dataUri = `data:image/jpeg;base64,${base64Crop}`;
747
-
748
- const translationResult = await translateImageContent(dataUri);
749
- if (translationResult.retryableFailure) {
750
- translateCooldownUntil = Date.now() + TRANSLATE_FAILURE_COOLDOWN_MS;
751
- console.warn(`Live translation cooldown active for ${TRANSLATE_FAILURE_COOLDOWN_MS / 1000}s after retryable API failure.`);
752
- } else {
753
- translateCooldownUntil = 0;
754
- }
755
-
756
- if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
757
- screenPickerWindow.webContents.send('vision-translate-result', translationResult.text);
758
- }
759
- }
760
- } catch (err) {
761
- console.error("Continuous translation loop failed:", err);
762
- } finally {
763
- isTranslateRequestInFlight = false;
764
- }
765
- }
766
-
767
- ipcMain.on('vision-cancel', () => {
768
- if (translateIntervalHandle) {
769
- clearInterval(translateIntervalHandle);
770
- translateIntervalHandle = null;
771
- }
772
- isTranslateRequestInFlight = false;
773
- translateCooldownUntil = 0;
774
- if (screenPickerWindow) screenPickerWindow.close();
775
- if (mainWindow) mainWindow.show();
776
- });
777
-
778
- // =====================
779
- // IPC Handlers — Proactive Assistant
780
- // =====================
781
- ipcMain.on('toggle-proactive', (event, isOn) => {
782
- if (isOn) {
783
- startProactiveLoop(); // reads interval from config automatically
784
- } else {
785
- stopProactiveLoop();
786
- }
787
- });
788
-
789
- ipcMain.on('record-behavior', (event, contextDescription) => {
790
- recordBehavior(contextDescription);
791
- });
792
-
793
- // Direct action executor for proactive Accept (no Gemini involved)
794
- ipcMain.handle('execute-proactive-action', async (event, action) => {
795
- if (!action || action.type === 'none') {
796
- return { success: false, message: 'ไม่มี action ที่จะดำเนินการค่ะ' };
797
- }
798
- try {
799
- const result = await executeAction(action);
800
- // Build a friendly confirmation message
801
- const messages = {
802
- open_url: `เปิดเว็บไซต์ให้แล้วค่ะ 🌐`,
803
- open_app: `เปิดแอป ${action.target} ให้แล้วค่ะ 🚀`,
804
- search: `ค้นหา "${action.target}" ให้แล้วค่ะ 🔍`,
805
- web_automation: result || 'ดำเนินการเสร็จแล้วค่ะ ✅',
806
- create_folder: `สร้างโฟลเดอร์ "${action.target}" แล้วค่ะ 📁`,
807
- clipboard_write: `คัดลอกข้อความแล้วค่ะ 📋`,
808
- learn_file: result || `เรียนรู้เอกสารเรียบร้อยค่ะ 📚`,
809
- };
810
- return {
811
- success: true,
812
- message: messages[action.type] || 'ดำเนินการเสร็จแล้วค่ะ ✅'
813
- };
814
- } catch (err) {
815
- console.error('[ProactiveAction] Error:', err);
816
- return { success: false, message: `เกิดข้อผิดพลาด: ${err.message}` };
817
- }
818
- });
819
-
820
- ipcMain.handle('capture-silent-screen', async () => {
821
- try {
822
- const primaryDisplay = screen.getPrimaryDisplay();
823
- const { width, height } = primaryDisplay.size;
824
-
825
- const sources = await desktopCapturer.getSources({
826
- types: ['screen'],
827
- thumbnailSize: { width, height }
828
- });
829
-
830
- const primarySource = sources[0];
831
- if (primarySource && primarySource.thumbnail) {
832
- return primarySource.thumbnail.toDataURL();
833
- }
834
- return null;
835
- } catch (err) {
836
- console.error("Error silently capturing screen:", err);
837
- return null;
838
- }
839
- });
840
-
841
- // =====================
842
- // Action Executor
843
- // =====================
844
- async function executeAction(action) {
845
- console.log("Executing action:", action);
846
-
847
- switch (action.type) {
848
- case 'open_url':
849
- openWebsite(action.target);
850
- break;
851
- case 'search':
852
- openSearch(action.target);
853
- break;
854
- case 'open_app':
855
- openApp(action.target);
856
- break;
857
- case 'web_automation':
858
- return await performWebAutomation(action.target);
859
- case 'create_folder':
860
- createFolder(action.target);
861
- break;
862
- case 'open_file':
863
- const fileRes = await openFile(action.target);
864
- return fileRes || `Successfully opened file: ${action.target} ✅`;
865
- case 'open_folder':
866
- const folderRes = await openFile(action.target);
867
- return folderRes || `Successfully opened folder: ${action.target} ✅`;
868
- case 'delete_file':
869
- await deleteFile(action.target);
870
- break;
871
- case 'find_path': {
872
- const result = findPath(action.target, {
873
- type: action.pathType,
874
- maxResults: 10
875
- });
876
- if (!result.success) {
877
- return result.message;
878
- }
879
-
880
- if (action.openAfter === true) {
881
- if (result.matches.length === 1) {
882
- const match = result.matches[0];
883
- const openResult = await openFile(match.path);
884
- return openResult || `Successfully found and opened ${match.type === 'dir' ? 'folder' : 'file'}: ${match.path} ✅`;
885
- }
886
- return `Found multiple matches for "${action.target}". Please be more specific:\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
887
- }
888
-
889
- return `Found matches for "${action.target}":\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
890
- }
891
- case 'clipboard_write':
892
- clipboard.writeText(action.target);
893
- break;
894
- case 'learn_file':
895
- return await indexFile(action.target);
896
- case 'learn_folder':
897
- const { indexFolder } = require('./src/AI_Brain/knowledge_base');
898
- return await indexFolder(action.target);
899
- case 'mcp_tool':
900
- const mcpResult = await mcpManager.callTool(action.server, action.target, action.args);
901
- return JSON.stringify(mcpResult.content);
902
- case 'mouse_move':
903
- return await granularAutomation.mouseMove(action.x, action.y);
904
- case 'mouse_click':
905
- return await granularAutomation.mouseClick(action.x, action.y, action.button || 1);
906
- case 'type_text':
907
- return await granularAutomation.typeText(action.target);
908
- case 'key_tap':
909
- return await granularAutomation.keyTap(action.target);
910
- case 'plugin':
911
- return await pluginManager.executePlugin(action.pluginName, action.target);
912
- case 'system_automation':
913
- return await handleSystemAutomation(action.target);
914
- }
915
- }
916
-
917
- async function handleSystemAutomation(target) {
918
- const [cmd, value] = target.split(':');
919
- switch (cmd) {
920
- case 'volume':
921
- return await SystemAutomation.setVolume(parseInt(value));
922
- case 'mute':
923
- return await SystemAutomation.mute();
924
- case 'brightness':
925
- return await SystemAutomation.setBrightness(parseInt(value));
926
- case 'sleep':
927
- return await SystemAutomation.sleep();
928
- case 'restart':
929
- return await SystemAutomation.restart();
930
- case 'shutdown':
931
- return await SystemAutomation.shutdown();
932
- case 'minimize_all':
933
- return await SystemAutomation.minimizeAll();
934
- default:
935
- // Handle cases where target is just the command (like 'sleep')
936
- if (SystemAutomation[target]) {
937
- return await SystemAutomation[target]();
938
- }
939
- throw new Error(`Unknown system automation command: ${target}`);
940
- }
941
- }