@pheem49/mint 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GUIDE_TH.md +113 -0
- package/README.md +239 -76
- package/assets/CLI_Screen.png +0 -0
- package/docs/assets/CLI_Screen.png +0 -0
- package/docs/guide.html +632 -0
- package/docs/index.html +5 -4
- package/main.js +66 -894
- package/mint-cli-logic.js +13 -1
- package/mint-cli.js +100 -9
- package/package.json +12 -4
- package/src/AI_Brain/Gemini_API.js +77 -20
- package/src/AI_Brain/autonomous_brain.js +10 -0
- package/src/AI_Brain/behavior_memory.js +26 -5
- package/src/AI_Brain/headless_agent.js +4 -0
- package/src/AI_Brain/knowledge_base.js +61 -8
- package/src/AI_Brain/memory_store.js +55 -7
- package/src/Automation_Layer/file_operations.js +1 -1
- package/src/CLI/chat_router.js +3 -2
- package/src/CLI/chat_ui.js +263 -838
- package/src/CLI/code_agent.js +144 -42
- package/src/CLI/gmail_auth.js +210 -0
- package/src/CLI/list_features.js +2 -0
- package/src/CLI/onboarding.js +307 -55
- package/src/CLI/updater.js +208 -0
- package/src/Channels/brave_search_bridge.js +35 -0
- package/src/Channels/discord_bridge.js +68 -0
- package/src/Channels/google_search_bridge.js +38 -0
- package/src/Channels/line_bridge.js +60 -0
- package/src/Channels/slack_bridge.js +53 -0
- package/src/Channels/telegram_bridge.js +49 -0
- package/src/Channels/whatsapp_bridge.js +55 -0
- package/src/Command_Parser/parser.js +12 -1
- package/src/Plugins/gmail.js +251 -0
- package/src/Plugins/google_calendar.js +245 -19
- package/src/Plugins/notion.js +256 -0
- package/src/System/action_executor.js +129 -0
- package/src/System/bridge_manager.js +76 -0
- package/src/System/chat_history_manager.js +23 -5
- package/src/System/config_manager.js +41 -7
- package/src/System/custom_workflows.js +31 -2
- package/src/System/google_tts_urls.js +51 -0
- package/src/System/ipc_handlers.js +238 -0
- package/src/System/proactive_loop.js +137 -0
- package/src/System/safety_manager.js +165 -0
- package/src/System/screen_capture.js +175 -0
- package/src/System/task_manager.js +15 -5
- package/src/System/window_manager.js +210 -0
- package/src/UI/renderer.js +33 -7
- package/src/UI/settings.html +24 -0
- package/src/UI/settings.js +14 -4
- package/src/UI/styles.css +14 -1
- package/tests/action_executor_safety.test.js +67 -0
- package/tests/gmail.test.js +135 -0
- package/tests/gmail_auth.test.js +129 -0
- package/tests/google_calendar.test.js +113 -0
- package/tests/google_tts_urls.test.js +24 -0
- package/tests/notion.test.js +121 -0
- package/tests/provider_routing.test.js +17 -1
- package/tests/safety_manager.test.js +40 -0
- package/tests/updater.test.js +32 -0
package/main.js
CHANGED
|
@@ -1,360 +1,82 @@
|
|
|
1
|
-
const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard
|
|
2
|
-
const path = require('path');
|
|
1
|
+
const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard } = require('electron');
|
|
3
2
|
require('dotenv').config();
|
|
4
3
|
|
|
5
4
|
const { handleChat, resetChat, getChatTranscript, translateImageContent, refreshApiKeyFromConfig } = require('./src/AI_Brain/Gemini_API');
|
|
6
|
-
const { openApp } = require('./src/Automation_Layer/open_app');
|
|
7
|
-
const { openWebsite, openSearch } = require('./src/Automation_Layer/open_website');
|
|
8
|
-
const { performWebAutomation } = require('./src/Automation_Layer/browser_automation');
|
|
9
|
-
const { createFolder, openFile, deleteFile, findPath } = require('./src/Automation_Layer/file_operations');
|
|
10
5
|
const { getSystemInfo, getWeather } = require('./src/System/system_info');
|
|
11
6
|
const { readConfig, writeConfig } = require('./src/System/config_manager');
|
|
12
7
|
const { parseCommand } = require('./src/Command_Parser/parser');
|
|
13
|
-
const
|
|
14
|
-
const {
|
|
15
|
-
const {
|
|
16
|
-
const {
|
|
8
|
+
const { executeAction } = require('./src/System/action_executor');
|
|
9
|
+
const { getGoogleTtsUrls } = require('./src/System/google_tts_urls');
|
|
10
|
+
const { createWindowManager } = require('./src/System/window_manager');
|
|
11
|
+
const { createProactiveLoop } = require('./src/System/proactive_loop');
|
|
12
|
+
const { createScreenCaptureController } = require('./src/System/screen_capture');
|
|
13
|
+
const { registerIpcHandlers } = require('./src/System/ipc_handlers');
|
|
17
14
|
|
|
18
|
-
const SystemAutomation = require('./src/System/system_automation');
|
|
19
15
|
const systemEvents = require('./src/System/system_events');
|
|
20
16
|
const customWorkflows = require('./src/System/custom_workflows');
|
|
21
17
|
const mcpManager = require('./src/Plugins/mcp_manager');
|
|
22
|
-
const granularAutomation = require('./src/System/granular_automation');
|
|
23
|
-
const googleTTS = require('google-tts-api');
|
|
24
18
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const behaviorSummary = getBehaviorSummary();
|
|
61
|
-
|
|
62
|
-
const result = await analyzeAndSuggest(base64Image, behaviorSummary);
|
|
63
|
-
|
|
64
|
-
if (result && result.message && Array.isArray(result.suggestions)) {
|
|
65
|
-
// Record the observed context into behavior memory
|
|
66
|
-
if (result.context) recordBehavior(result.context);
|
|
67
|
-
|
|
68
|
-
// Push suggestion to the renderer
|
|
69
|
-
if (mainWindow && !mainWindow.isDestroyed()) {
|
|
70
|
-
mainWindow.webContents.send('proactive-suggestion', result);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Hide Glow after analysis
|
|
75
|
-
hideProactiveGlow();
|
|
76
|
-
} catch (err) {
|
|
77
|
-
console.error('[Proactive] Cycle error:', err.message);
|
|
78
|
-
hideProactiveGlow();
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function createProactiveGlowWindow() {
|
|
83
|
-
if (proactiveGlowWindow) return;
|
|
84
|
-
const { width, height } = screen.getPrimaryDisplay().bounds;
|
|
85
|
-
|
|
86
|
-
proactiveGlowWindow = new BrowserWindow({
|
|
87
|
-
width, height,
|
|
88
|
-
x: 0, y: 0,
|
|
89
|
-
frame: false,
|
|
90
|
-
transparent: true,
|
|
91
|
-
alwaysOnTop: true,
|
|
92
|
-
skipTaskbar: true,
|
|
93
|
-
focusable: false,
|
|
94
|
-
show: false,
|
|
95
|
-
webPreferences: {
|
|
96
|
-
nodeIntegration: false,
|
|
97
|
-
contextIsolation: true
|
|
98
|
-
}
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
proactiveGlowWindow.setIgnoreMouseEvents(true);
|
|
102
|
-
proactiveGlowWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
|
|
103
|
-
proactiveGlowWindow.setAlwaysOnTop(true, 'screen-saver');
|
|
104
|
-
proactiveGlowWindow.loadFile('src/UI/proactive-glow.html');
|
|
105
|
-
|
|
106
|
-
proactiveGlowWindow.on('closed', () => { proactiveGlowWindow = null; });
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
function showProactiveGlow() {
|
|
110
|
-
if (!proactiveGlowWindow) createProactiveGlowWindow();
|
|
111
|
-
if (proactiveGlowWindow) proactiveGlowWindow.showInactive();
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function hideProactiveGlow() {
|
|
115
|
-
if (proactiveGlowWindow && !proactiveGlowWindow.isDestroyed()) {
|
|
116
|
-
proactiveGlowWindow.hide();
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
function startProactiveLoop(intervalSec) {
|
|
121
|
-
// Stop existing loop first
|
|
122
|
-
if (proactiveIntervalHandle) {
|
|
123
|
-
clearInterval(proactiveIntervalHandle);
|
|
124
|
-
proactiveIntervalHandle = null;
|
|
125
|
-
}
|
|
126
|
-
// Read from config if not provided
|
|
127
|
-
const cfg = readConfig();
|
|
128
|
-
const ms = (intervalSec || cfg.proactiveInterval || 60) * 1000;
|
|
129
|
-
console.log(`[Proactive] Starting loop — interval: ${ms / 1000}s`);
|
|
130
|
-
proactiveIntervalHandle = setInterval(runProactiveCycle, ms);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function stopProactiveLoop() {
|
|
134
|
-
if (proactiveIntervalHandle) {
|
|
135
|
-
clearInterval(proactiveIntervalHandle);
|
|
136
|
-
proactiveIntervalHandle = null;
|
|
137
|
-
console.log('[Proactive] Stopped proactive loop.');
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
// Check idle state every minute to pause/resume background tasks
|
|
142
|
-
const IDLE_THRESHOLD_SEC = 300; // 5 minutes
|
|
143
|
-
setInterval(() => {
|
|
144
|
-
if (!proactiveIntervalHandle) return; // Only manage if it's supposed to be active (Smart Context is ON)
|
|
145
|
-
|
|
146
|
-
// powerMonitor.getSystemIdleTime() is available after app is ready
|
|
147
|
-
if (app.isReady()) {
|
|
148
|
-
const idleSec = powerMonitor.getSystemIdleTime();
|
|
149
|
-
if (idleSec >= IDLE_THRESHOLD_SEC) {
|
|
150
|
-
console.log(`[System Idle] User idle for ${idleSec}s. Pausing Proactive loop to save resources.`);
|
|
151
|
-
stopProactiveLoop();
|
|
152
|
-
|
|
153
|
-
// Wait for user to come back
|
|
154
|
-
const resumeChecker = setInterval(() => {
|
|
155
|
-
if (powerMonitor.getSystemIdleTime() < 10) {
|
|
156
|
-
console.log('[System Idle] User returned. Resuming Proactive loop.');
|
|
157
|
-
clearInterval(resumeChecker);
|
|
158
|
-
startProactiveLoop();
|
|
159
|
-
}
|
|
160
|
-
}, 5000);
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
}, 60000);
|
|
164
|
-
|
|
165
|
-
function createWindow() {
|
|
166
|
-
mainWindow = new BrowserWindow({
|
|
167
|
-
width: 600,
|
|
168
|
-
height: 800,
|
|
169
|
-
icon: path.join(__dirname, 'assets', 'icon.png'),
|
|
170
|
-
webPreferences: {
|
|
171
|
-
preload: path.join(__dirname, 'preload.js'),
|
|
172
|
-
nodeIntegration: false,
|
|
173
|
-
contextIsolation: true,
|
|
174
|
-
},
|
|
175
|
-
frame: false,
|
|
176
|
-
transparent: true,
|
|
177
|
-
resizable: true,
|
|
178
|
-
show: false
|
|
179
|
-
});
|
|
180
|
-
|
|
181
|
-
mainWindow.loadFile('src/UI/index.html');
|
|
182
|
-
|
|
183
|
-
mainWindow.on('ready-to-show', () => {
|
|
184
|
-
mainWindow.show();
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
mainWindow.on('close', function (event) {
|
|
188
|
-
if (!app.isQuiting) {
|
|
189
|
-
event.preventDefault();
|
|
190
|
-
mainWindow.hide();
|
|
191
|
-
}
|
|
192
|
-
return false;
|
|
193
|
-
});
|
|
194
|
-
|
|
195
|
-
mainWindow.on('focus', () => {
|
|
196
|
-
// clearFloatingUnread(); // Disabled
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
function createTray() {
|
|
201
|
-
const iconPath = path.join(__dirname, 'assets', 'icon.png');
|
|
202
|
-
let icon = nativeImage.createFromPath(iconPath);
|
|
203
|
-
// Optional: resize it for the tray if needed, but Electron handles this natively on many OSs
|
|
204
|
-
icon = icon.resize({ width: 16, height: 16 });
|
|
205
|
-
|
|
206
|
-
tray = new Tray(icon);
|
|
207
|
-
tray.setToolTip('Mint AI Assistant');
|
|
208
|
-
|
|
209
|
-
const contextMenu = Menu.buildFromTemplate([
|
|
210
|
-
{ label: 'Show App', click: () => { if (mainWindow) mainWindow.show(); } },
|
|
211
|
-
{ label: 'Settings', click: () => { createSettingsWindow(); } },
|
|
212
|
-
{ type: 'separator' },
|
|
213
|
-
{ label: 'Quit', click: () => {
|
|
214
|
-
app.isQuiting = true;
|
|
215
|
-
app.quit();
|
|
216
|
-
}}
|
|
217
|
-
]);
|
|
218
|
-
|
|
219
|
-
tray.setContextMenu(contextMenu);
|
|
220
|
-
|
|
221
|
-
tray.on('click', () => {
|
|
222
|
-
if (mainWindow) {
|
|
223
|
-
if (mainWindow.isVisible()) {
|
|
224
|
-
mainWindow.hide();
|
|
225
|
-
} else {
|
|
226
|
-
mainWindow.show();
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
function createSettingsWindow() {
|
|
233
|
-
if (settingsWindow) {
|
|
234
|
-
settingsWindow.focus();
|
|
235
|
-
return;
|
|
236
|
-
}
|
|
237
|
-
settingsWindow = new BrowserWindow({
|
|
238
|
-
width: 720,
|
|
239
|
-
height: 620,
|
|
240
|
-
minWidth: 640,
|
|
241
|
-
minHeight: 560,
|
|
242
|
-
icon: path.join(__dirname, 'assets', 'icon.png'),
|
|
243
|
-
webPreferences: {
|
|
244
|
-
preload: path.join(__dirname, 'preload-settings.js'),
|
|
245
|
-
nodeIntegration: false,
|
|
246
|
-
contextIsolation: true,
|
|
247
|
-
},
|
|
248
|
-
frame: false,
|
|
249
|
-
transparent: true,
|
|
250
|
-
resizable: true,
|
|
251
|
-
parent: mainWindow,
|
|
252
|
-
});
|
|
253
|
-
settingsWindow.loadFile('src/UI/settings.html');
|
|
254
|
-
settingsWindow.on('closed', () => { settingsWindow = null; });
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
function createSpotlightWindow() {
|
|
258
|
-
if (spotlightWindow) {
|
|
259
|
-
spotlightWindow.show();
|
|
260
|
-
return;
|
|
19
|
+
const projectRoot = __dirname;
|
|
20
|
+
const windowManager = createWindowManager(projectRoot);
|
|
21
|
+
const proactiveLoop = createProactiveLoop({
|
|
22
|
+
app,
|
|
23
|
+
projectRoot,
|
|
24
|
+
readConfig,
|
|
25
|
+
getMainWindow: windowManager.getMainWindow
|
|
26
|
+
});
|
|
27
|
+
const screenCapture = createScreenCaptureController({
|
|
28
|
+
projectRoot,
|
|
29
|
+
translateImageContent,
|
|
30
|
+
getMainWindow: windowManager.getMainWindow
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
registerIpcHandlers({
|
|
34
|
+
app,
|
|
35
|
+
ipcMain,
|
|
36
|
+
shell,
|
|
37
|
+
clipboard,
|
|
38
|
+
windowManager,
|
|
39
|
+
proactiveLoop,
|
|
40
|
+
screenCapture,
|
|
41
|
+
services: {
|
|
42
|
+
handleChat,
|
|
43
|
+
resetChat,
|
|
44
|
+
getChatTranscript,
|
|
45
|
+
refreshApiKeyFromConfig,
|
|
46
|
+
getSystemInfo,
|
|
47
|
+
getWeather,
|
|
48
|
+
readConfig,
|
|
49
|
+
writeConfig,
|
|
50
|
+
parseCommand,
|
|
51
|
+
executeAction,
|
|
52
|
+
getGoogleTtsUrls,
|
|
53
|
+
customWorkflows
|
|
261
54
|
}
|
|
262
|
-
|
|
263
|
-
const { width: screenWidth, height: screenHeight } = screen.getPrimaryDisplay().workAreaSize;
|
|
264
|
-
const windowWidth = 600;
|
|
265
|
-
const windowHeight = 80; // Starts small, expands if results show
|
|
266
|
-
|
|
267
|
-
spotlightWindow = new BrowserWindow({
|
|
268
|
-
width: windowWidth,
|
|
269
|
-
height: windowHeight,
|
|
270
|
-
x: Math.floor((screenWidth - windowWidth) / 2),
|
|
271
|
-
y: Math.floor(screenHeight * 0.25), // 25% from top
|
|
272
|
-
frame: false,
|
|
273
|
-
transparent: true,
|
|
274
|
-
alwaysOnTop: true,
|
|
275
|
-
skipTaskbar: true,
|
|
276
|
-
show: false,
|
|
277
|
-
webPreferences: {
|
|
278
|
-
preload: path.join(__dirname, 'src/UI/preload-spotlight.js'),
|
|
279
|
-
nodeIntegration: false,
|
|
280
|
-
contextIsolation: true,
|
|
281
|
-
}
|
|
282
|
-
});
|
|
283
|
-
|
|
284
|
-
spotlightWindow.loadFile('src/UI/spotlight.html');
|
|
285
|
-
|
|
286
|
-
spotlightWindow.on('blur', () => {
|
|
287
|
-
spotlightWindow.hide();
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
spotlightWindow.on('closed', () => {
|
|
291
|
-
spotlightWindow = null;
|
|
292
|
-
});
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
// Floating window logic removed
|
|
296
|
-
|
|
297
|
-
function createWidgetWindow() {
|
|
298
|
-
if (widgetWindow) return;
|
|
299
|
-
|
|
300
|
-
widgetWindow = new BrowserWindow({
|
|
301
|
-
width: 150,
|
|
302
|
-
height: 150,
|
|
303
|
-
frame: false,
|
|
304
|
-
transparent: true,
|
|
305
|
-
resizable: false,
|
|
306
|
-
alwaysOnTop: true,
|
|
307
|
-
skipTaskbar: true,
|
|
308
|
-
show: true,
|
|
309
|
-
webPreferences: {
|
|
310
|
-
preload: require('path').join(__dirname, 'src/UI/preload-widget.js'),
|
|
311
|
-
nodeIntegration: false,
|
|
312
|
-
contextIsolation: true
|
|
313
|
-
}
|
|
314
|
-
});
|
|
315
|
-
|
|
316
|
-
widgetWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
|
|
317
|
-
widgetWindow.setAlwaysOnTop(true, 'floating');
|
|
318
|
-
|
|
319
|
-
// Position Top-Right loosely
|
|
320
|
-
try {
|
|
321
|
-
const primaryDisplay = require('electron').screen.getPrimaryDisplay();
|
|
322
|
-
const { width, height, x, y } = primaryDisplay.workArea;
|
|
323
|
-
widgetWindow.setPosition(x + width - 150 - 40, y + 40);
|
|
324
|
-
} catch (e) {}
|
|
325
|
-
|
|
326
|
-
widgetWindow.loadFile('src/UI/widget.html');
|
|
327
|
-
|
|
328
|
-
widgetWindow.on('closed', () => {
|
|
329
|
-
widgetWindow = null;
|
|
330
|
-
});
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
// Floating unread logic removed
|
|
55
|
+
});
|
|
334
56
|
|
|
335
57
|
app.whenReady().then(() => {
|
|
336
58
|
const config = readConfig();
|
|
337
|
-
|
|
338
|
-
createTray();
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
// Only show AI widget if enabled in settings
|
|
59
|
+
const mainWindow = windowManager.createMainWindow();
|
|
60
|
+
windowManager.createTray();
|
|
61
|
+
|
|
342
62
|
if (config.showDesktopWidget !== false) {
|
|
343
|
-
createWidgetWindow();
|
|
63
|
+
windowManager.createWidgetWindow();
|
|
344
64
|
}
|
|
345
65
|
|
|
346
|
-
// Initialize MCP Servers
|
|
347
66
|
mcpManager.init().catch(err => console.error('[MCP] Init Error:', err));
|
|
348
|
-
|
|
349
|
-
|
|
67
|
+
|
|
68
|
+
const bridgeManager = require('./src/System/bridge_manager');
|
|
69
|
+
bridgeManager.init().catch(err => console.error('[BridgeManager] Init Error:', err));
|
|
70
|
+
|
|
350
71
|
systemEvents.startMonitoring();
|
|
351
72
|
if (config.enableCustomWorkflows !== false) {
|
|
352
73
|
customWorkflows.startMonitoring(mainWindow.webContents);
|
|
353
74
|
}
|
|
354
75
|
|
|
355
76
|
systemEvents.on('low-battery', (level) => {
|
|
356
|
-
|
|
357
|
-
|
|
77
|
+
const currentMainWindow = windowManager.getMainWindow();
|
|
78
|
+
if (currentMainWindow && !currentMainWindow.isDestroyed()) {
|
|
79
|
+
currentMainWindow.webContents.send('proactive-notification', {
|
|
358
80
|
message: `⚠️ Battery is low (${level}%). Please plug in your charger. ✨`,
|
|
359
81
|
type: 'warning'
|
|
360
82
|
});
|
|
@@ -362,580 +84,30 @@ app.whenReady().then(() => {
|
|
|
362
84
|
});
|
|
363
85
|
|
|
364
86
|
systemEvents.on('connection-change', (isOnline) => {
|
|
365
|
-
|
|
87
|
+
const currentMainWindow = windowManager.getMainWindow();
|
|
88
|
+
if (currentMainWindow && !currentMainWindow.isDestroyed()) {
|
|
366
89
|
const msg = isOnline ? '✅ Internet connection restored. ✨' : '❌ Internet connection lost.';
|
|
367
|
-
|
|
90
|
+
currentMainWindow.webContents.send('proactive-notification', { message: msg, type: 'info' });
|
|
368
91
|
}
|
|
369
92
|
});
|
|
370
93
|
|
|
371
|
-
globalShortcut.register('CommandOrControl+Shift+Space',
|
|
372
|
-
|
|
373
|
-
if (mainWindow.isVisible()) {
|
|
374
|
-
mainWindow.hide();
|
|
375
|
-
} else {
|
|
376
|
-
mainWindow.show();
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
});
|
|
94
|
+
globalShortcut.register('CommandOrControl+Shift+Space', windowManager.toggleMainWindow);
|
|
95
|
+
globalShortcut.register('Alt+Space', windowManager.toggleSpotlightWindow);
|
|
380
96
|
|
|
381
|
-
|
|
382
|
-
if (spotlightWindow && spotlightWindow.isVisible()) {
|
|
383
|
-
spotlightWindow.hide();
|
|
384
|
-
} else {
|
|
385
|
-
createSpotlightWindow();
|
|
386
|
-
spotlightWindow.show();
|
|
387
|
-
}
|
|
388
|
-
});
|
|
97
|
+
proactiveLoop.startIdleWatcher();
|
|
389
98
|
|
|
390
|
-
app.on('activate',
|
|
391
|
-
if (BrowserWindow.getAllWindows().length === 0)
|
|
99
|
+
app.on('activate', () => {
|
|
100
|
+
if (BrowserWindow.getAllWindows().length === 0) {
|
|
101
|
+
windowManager.createMainWindow();
|
|
102
|
+
}
|
|
392
103
|
});
|
|
393
104
|
});
|
|
394
105
|
|
|
395
|
-
app.on('window-all-closed', (
|
|
396
|
-
|
|
106
|
+
app.on('window-all-closed', (event) => {
|
|
107
|
+
event.preventDefault();
|
|
397
108
|
});
|
|
398
109
|
|
|
399
110
|
app.on('will-quit', () => {
|
|
400
111
|
globalShortcut.unregisterAll();
|
|
401
112
|
mcpManager.shutdown();
|
|
402
113
|
});
|
|
403
|
-
|
|
404
|
-
// =====================
|
|
405
|
-
// IPC Handlers — Chat
|
|
406
|
-
// =====================
|
|
407
|
-
ipcMain.handle('chat-message', async (event, message, base64Image = null, base64Audio = null) => {
|
|
408
|
-
try {
|
|
409
|
-
const rawResponse = await handleChat(message, base64Image, base64Audio);
|
|
410
|
-
const aiResponse = parseCommand(rawResponse);
|
|
411
|
-
|
|
412
|
-
if (aiResponse.action && aiResponse.action.type !== 'none') {
|
|
413
|
-
try {
|
|
414
|
-
const actionResult = await executeAction(aiResponse.action);
|
|
415
|
-
// If the action returned a string result (e.g. from Web Automation), append it.
|
|
416
|
-
if (actionResult && typeof actionResult === 'string') {
|
|
417
|
-
aiResponse.response += `\n\n${actionResult}`;
|
|
418
|
-
}
|
|
419
|
-
} catch (err) {
|
|
420
|
-
console.error("Action execution error:", err);
|
|
421
|
-
aiResponse.response += "\n\n(Note: I tried to execute the action, but an error occurred.)";
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
return aiResponse;
|
|
426
|
-
} catch (error) {
|
|
427
|
-
console.error('Chat error:', error);
|
|
428
|
-
return { response: 'Error communicating with Gemini API. Check your console and API key.', action: { type: 'none' } };
|
|
429
|
-
}
|
|
430
|
-
});
|
|
431
|
-
|
|
432
|
-
ipcMain.on('close-window', (event) => {
|
|
433
|
-
if (mainWindow) mainWindow.hide();
|
|
434
|
-
});
|
|
435
|
-
|
|
436
|
-
ipcMain.on('minimize-window', (event) => {
|
|
437
|
-
if (mainWindow) mainWindow.minimize();
|
|
438
|
-
});
|
|
439
|
-
|
|
440
|
-
ipcMain.on('quit-app', (event) => {
|
|
441
|
-
app.isQuiting = true;
|
|
442
|
-
app.quit();
|
|
443
|
-
});
|
|
444
|
-
|
|
445
|
-
ipcMain.on('maximize-window', (event) => {
|
|
446
|
-
if (mainWindow) {
|
|
447
|
-
if (mainWindow.isMaximized()) {
|
|
448
|
-
mainWindow.unmaximize();
|
|
449
|
-
} else {
|
|
450
|
-
mainWindow.maximize();
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
});
|
|
454
|
-
|
|
455
|
-
ipcMain.handle('reset-chat', () => {
|
|
456
|
-
resetChat();
|
|
457
|
-
return { success: true };
|
|
458
|
-
});
|
|
459
|
-
|
|
460
|
-
ipcMain.handle('get-chat-history', () => {
|
|
461
|
-
return getChatTranscript();
|
|
462
|
-
});
|
|
463
|
-
|
|
464
|
-
// =====================
|
|
465
|
-
// IPC Handlers — Settings
|
|
466
|
-
// =====================
|
|
467
|
-
ipcMain.handle('open-settings', () => {
|
|
468
|
-
createSettingsWindow();
|
|
469
|
-
});
|
|
470
|
-
|
|
471
|
-
ipcMain.handle('get-settings', () => {
|
|
472
|
-
return readConfig();
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
ipcMain.handle('save-settings', (event, config) => {
|
|
476
|
-
console.log('[Settings] Saving new config. MCP Servers count:', Object.keys(config.mcpServers || {}).length);
|
|
477
|
-
const result = writeConfig(config);
|
|
478
|
-
// Refresh API key if user updated it in settings
|
|
479
|
-
refreshApiKeyFromConfig();
|
|
480
|
-
// 🔔 Notify main chat window immediately
|
|
481
|
-
if (mainWindow && !mainWindow.isDestroyed()) {
|
|
482
|
-
mainWindow.webContents.send('settings-changed', config);
|
|
483
|
-
}
|
|
484
|
-
// Restart proactive loop with new interval if it's running
|
|
485
|
-
if (proactiveIntervalHandle) {
|
|
486
|
-
startProactiveLoop(config.proactiveInterval);
|
|
487
|
-
}
|
|
488
|
-
if (config.enableCustomWorkflows !== false) {
|
|
489
|
-
customWorkflows.startMonitoring(mainWindow.webContents);
|
|
490
|
-
} else {
|
|
491
|
-
customWorkflows.stopMonitoring();
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
// Toggle Desktop Widget
|
|
495
|
-
if (config.showDesktopWidget === false) {
|
|
496
|
-
if (widgetWindow && !widgetWindow.isDestroyed()) {
|
|
497
|
-
widgetWindow.close();
|
|
498
|
-
widgetWindow = null;
|
|
499
|
-
}
|
|
500
|
-
} else {
|
|
501
|
-
if (!widgetWindow || widgetWindow.isDestroyed()) {
|
|
502
|
-
createWidgetWindow();
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
return result;
|
|
507
|
-
});
|
|
508
|
-
|
|
509
|
-
ipcMain.on('set-ai-state', (event, state) => {
|
|
510
|
-
if (widgetWindow && !widgetWindow.isDestroyed()) {
|
|
511
|
-
widgetWindow.webContents.send('widget-state', state);
|
|
512
|
-
}
|
|
513
|
-
});
|
|
514
|
-
|
|
515
|
-
ipcMain.on('close-settings', () => {
|
|
516
|
-
if (settingsWindow) settingsWindow.close();
|
|
517
|
-
});
|
|
518
|
-
|
|
519
|
-
ipcMain.on('quit-app', () => {
|
|
520
|
-
app.isQuiting = true;
|
|
521
|
-
app.quit();
|
|
522
|
-
});
|
|
523
|
-
|
|
524
|
-
ipcMain.handle('open-custom-workflows', () => {
|
|
525
|
-
customWorkflows.openConfigFile();
|
|
526
|
-
});
|
|
527
|
-
|
|
528
|
-
ipcMain.handle('reload-custom-workflows', () => {
|
|
529
|
-
customWorkflows.loadWorkflows();
|
|
530
|
-
return { success: true };
|
|
531
|
-
});
|
|
532
|
-
|
|
533
|
-
// =====================
|
|
534
|
-
// IPC Handlers — Spotlight
|
|
535
|
-
// =====================
|
|
536
|
-
ipcMain.on('spotlight-close', () => {
|
|
537
|
-
if (spotlightWindow) spotlightWindow.close();
|
|
538
|
-
});
|
|
539
|
-
|
|
540
|
-
ipcMain.on('spotlight-hide', () => {
|
|
541
|
-
if (spotlightWindow) spotlightWindow.hide();
|
|
542
|
-
});
|
|
543
|
-
|
|
544
|
-
ipcMain.on('spotlight-submit', async (event, query) => {
|
|
545
|
-
console.log('[Spotlight] Submit:', query);
|
|
546
|
-
if (spotlightWindow) spotlightWindow.hide();
|
|
547
|
-
|
|
548
|
-
// Show main window and send the message
|
|
549
|
-
if (mainWindow) {
|
|
550
|
-
mainWindow.show();
|
|
551
|
-
// We can either just show it or actually trigger the chat
|
|
552
|
-
mainWindow.webContents.send('spotlight-to-chat', query);
|
|
553
|
-
}
|
|
554
|
-
});
|
|
555
|
-
|
|
556
|
-
ipcMain.on('spotlight-resize', (event, width, height) => {
|
|
557
|
-
if (spotlightWindow) {
|
|
558
|
-
spotlightWindow.setSize(width, height);
|
|
559
|
-
}
|
|
560
|
-
});
|
|
561
|
-
|
|
562
|
-
// Floating IPC removed
|
|
563
|
-
|
|
564
|
-
ipcMain.handle('open-external', (event, url) => {
|
|
565
|
-
shell.openExternal(url);
|
|
566
|
-
});
|
|
567
|
-
|
|
568
|
-
// =====================
|
|
569
|
-
// IPC Handlers — Clipboard
|
|
570
|
-
// =====================
|
|
571
|
-
ipcMain.handle('clipboard-read', () => {
|
|
572
|
-
return clipboard.readText();
|
|
573
|
-
});
|
|
574
|
-
|
|
575
|
-
ipcMain.handle('clipboard-write', (event, text) => {
|
|
576
|
-
clipboard.writeText(text);
|
|
577
|
-
return { success: true };
|
|
578
|
-
});
|
|
579
|
-
|
|
580
|
-
// =====================
|
|
581
|
-
// IPC Handlers — TTS
|
|
582
|
-
// =====================
|
|
583
|
-
ipcMain.handle('get-tts-urls', async (event, text) => {
|
|
584
|
-
try {
|
|
585
|
-
const isThai = /[\u0E00-\u0E7F]/.test(text);
|
|
586
|
-
const lang = isThai ? 'th' : 'en';
|
|
587
|
-
|
|
588
|
-
const results = googleTTS.getAllAudioUrls(text, {
|
|
589
|
-
lang: lang,
|
|
590
|
-
slow: false,
|
|
591
|
-
host: 'https://translate.google.com',
|
|
592
|
-
splitPunct: ',.?!;:',
|
|
593
|
-
});
|
|
594
|
-
return results;
|
|
595
|
-
} catch (e) {
|
|
596
|
-
console.error("TTS Error:", e);
|
|
597
|
-
return [];
|
|
598
|
-
}
|
|
599
|
-
});
|
|
600
|
-
|
|
601
|
-
// =====================
|
|
602
|
-
// IPC Handlers — System Info
|
|
603
|
-
// =====================
|
|
604
|
-
ipcMain.handle('get-system-info', async () => {
|
|
605
|
-
return getSystemInfo();
|
|
606
|
-
});
|
|
607
|
-
|
|
608
|
-
ipcMain.handle('get-weather', async (event, city) => {
|
|
609
|
-
return getWeather(city);
|
|
610
|
-
});
|
|
611
|
-
|
|
612
|
-
// =====================
|
|
613
|
-
// IPC Handlers — Screen Vision
|
|
614
|
-
// =====================
|
|
615
|
-
ipcMain.handle('start-screen-capture', async () => {
|
|
616
|
-
if (screenPickerWindow) return; // Prevent multiple windows
|
|
617
|
-
|
|
618
|
-
try {
|
|
619
|
-
// Capture primary display
|
|
620
|
-
const primaryDisplay = screen.getPrimaryDisplay();
|
|
621
|
-
const { width, height } = primaryDisplay.size;
|
|
622
|
-
|
|
623
|
-
const sources = await desktopCapturer.getSources({
|
|
624
|
-
types: ['screen'],
|
|
625
|
-
thumbnailSize: { width, height }
|
|
626
|
-
});
|
|
627
|
-
|
|
628
|
-
// Find the full primary screen (usually "Screen 1" or "Entire screen")
|
|
629
|
-
const primarySource = sources[0]; // Assuming single/primary monitor is the first
|
|
630
|
-
|
|
631
|
-
// Create transparent, borderless screen picker window
|
|
632
|
-
screenPickerWindow = new BrowserWindow({
|
|
633
|
-
width, height,
|
|
634
|
-
x: primaryDisplay.bounds.x, y: primaryDisplay.bounds.y,
|
|
635
|
-
fullscreen: true,
|
|
636
|
-
transparent: true,
|
|
637
|
-
frame: false,
|
|
638
|
-
alwaysOnTop: true,
|
|
639
|
-
skipTaskbar: true,
|
|
640
|
-
webPreferences: {
|
|
641
|
-
preload: path.join(__dirname, 'preload-picker.js'),
|
|
642
|
-
nodeIntegration: false,
|
|
643
|
-
contextIsolation: true
|
|
644
|
-
}
|
|
645
|
-
});
|
|
646
|
-
|
|
647
|
-
await screenPickerWindow.loadFile('src/UI/screenPicker.html');
|
|
648
|
-
|
|
649
|
-
// Ensure image data isn't missing
|
|
650
|
-
if(primarySource && primarySource.thumbnail) {
|
|
651
|
-
screenPickerWindow.webContents.send('screenshot-data', primarySource.thumbnail.toDataURL());
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
screenPickerWindow.on('closed', () => { screenPickerWindow = null; });
|
|
655
|
-
} catch (err) {
|
|
656
|
-
console.error("Error starting screen capture:", err);
|
|
657
|
-
}
|
|
658
|
-
});
|
|
659
|
-
|
|
660
|
-
// Received selection from the picker
|
|
661
|
-
ipcMain.on('vision-selection', (event, base64Image) => {
|
|
662
|
-
if (screenPickerWindow) screenPickerWindow.close();
|
|
663
|
-
|
|
664
|
-
// Relay image back to main window interface
|
|
665
|
-
if (mainWindow) {
|
|
666
|
-
mainWindow.webContents.send('vision-ready', base64Image);
|
|
667
|
-
mainWindow.show(); // Bring chat back to focus
|
|
668
|
-
}
|
|
669
|
-
});
|
|
670
|
-
|
|
671
|
-
let translateIntervalHandle = null;
|
|
672
|
-
let isTranslateRequestInFlight = false;
|
|
673
|
-
let translateCooldownUntil = 0;
|
|
674
|
-
const TRANSLATE_REFRESH_MS = 3000;
|
|
675
|
-
const TRANSLATE_FAILURE_COOLDOWN_MS = 15000;
|
|
676
|
-
|
|
677
|
-
// Start Continuous Live Translate
|
|
678
|
-
ipcMain.on('vision-translate-start', async (event, rect) => {
|
|
679
|
-
if (!screenPickerWindow) return;
|
|
680
|
-
|
|
681
|
-
screenPickerWindow.setIgnoreMouseEvents(true, { forward: true });
|
|
682
|
-
isTranslateRequestInFlight = false;
|
|
683
|
-
translateCooldownUntil = 0;
|
|
684
|
-
|
|
685
|
-
// Stop any existing loop
|
|
686
|
-
if (translateIntervalHandle) clearInterval(translateIntervalHandle);
|
|
687
|
-
|
|
688
|
-
// Initial capture immediately
|
|
689
|
-
captureAndTranslate(rect);
|
|
690
|
-
|
|
691
|
-
// Then start ticking every 3 seconds
|
|
692
|
-
translateIntervalHandle = setInterval(() => {
|
|
693
|
-
captureAndTranslate(rect);
|
|
694
|
-
}, TRANSLATE_REFRESH_MS);
|
|
695
|
-
});
|
|
696
|
-
|
|
697
|
-
// Stop Continuous Live Translate
|
|
698
|
-
ipcMain.on('vision-translate-stop', () => {
|
|
699
|
-
if (translateIntervalHandle) {
|
|
700
|
-
clearInterval(translateIntervalHandle);
|
|
701
|
-
translateIntervalHandle = null;
|
|
702
|
-
}
|
|
703
|
-
if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
|
|
704
|
-
screenPickerWindow.setIgnoreMouseEvents(false);
|
|
705
|
-
}
|
|
706
|
-
isTranslateRequestInFlight = false;
|
|
707
|
-
translateCooldownUntil = 0;
|
|
708
|
-
});
|
|
709
|
-
|
|
710
|
-
ipcMain.on('vision-overlay-interactable', (event, isInteractable) => {
|
|
711
|
-
if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
|
|
712
|
-
screenPickerWindow.setIgnoreMouseEvents(!isInteractable, { forward: true });
|
|
713
|
-
});
|
|
714
|
-
|
|
715
|
-
async function captureAndTranslate(rect) {
|
|
716
|
-
if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
|
|
717
|
-
if (isTranslateRequestInFlight) return;
|
|
718
|
-
if (Date.now() < translateCooldownUntil) return;
|
|
719
|
-
|
|
720
|
-
try {
|
|
721
|
-
isTranslateRequestInFlight = true;
|
|
722
|
-
const primaryDisplay = screen.getPrimaryDisplay();
|
|
723
|
-
const sources = await desktopCapturer.getSources({
|
|
724
|
-
types: ['screen'],
|
|
725
|
-
thumbnailSize: {
|
|
726
|
-
width: primaryDisplay.size.width,
|
|
727
|
-
height: primaryDisplay.size.height
|
|
728
|
-
}
|
|
729
|
-
});
|
|
730
|
-
|
|
731
|
-
if (sources.length > 0) {
|
|
732
|
-
const screenImage = sources[0].thumbnail;
|
|
733
|
-
|
|
734
|
-
// Crop the specific rect out of the full screen image
|
|
735
|
-
// We use standard scale (assuming 1:1 scale for exact rect, although hidpi might vary.
|
|
736
|
-
// In a robust implementation we might need `Math.floor(rect.x * display.scaleFactor)`)
|
|
737
|
-
const croppedImage = screenImage.crop({
|
|
738
|
-
x: Math.round(rect.x),
|
|
739
|
-
y: Math.round(rect.y),
|
|
740
|
-
width: Math.round(rect.width),
|
|
741
|
-
height: Math.round(rect.height)
|
|
742
|
-
});
|
|
743
|
-
|
|
744
|
-
// Convert to JPEG Base64 for faster processing
|
|
745
|
-
const base64Crop = croppedImage.toJPEG(70).toString('base64');
|
|
746
|
-
const dataUri = `data:image/jpeg;base64,${base64Crop}`;
|
|
747
|
-
|
|
748
|
-
const translationResult = await translateImageContent(dataUri);
|
|
749
|
-
if (translationResult.retryableFailure) {
|
|
750
|
-
translateCooldownUntil = Date.now() + TRANSLATE_FAILURE_COOLDOWN_MS;
|
|
751
|
-
console.warn(`Live translation cooldown active for ${TRANSLATE_FAILURE_COOLDOWN_MS / 1000}s after retryable API failure.`);
|
|
752
|
-
} else {
|
|
753
|
-
translateCooldownUntil = 0;
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
|
|
757
|
-
screenPickerWindow.webContents.send('vision-translate-result', translationResult.text);
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
} catch (err) {
|
|
761
|
-
console.error("Continuous translation loop failed:", err);
|
|
762
|
-
} finally {
|
|
763
|
-
isTranslateRequestInFlight = false;
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
ipcMain.on('vision-cancel', () => {
|
|
768
|
-
if (translateIntervalHandle) {
|
|
769
|
-
clearInterval(translateIntervalHandle);
|
|
770
|
-
translateIntervalHandle = null;
|
|
771
|
-
}
|
|
772
|
-
isTranslateRequestInFlight = false;
|
|
773
|
-
translateCooldownUntil = 0;
|
|
774
|
-
if (screenPickerWindow) screenPickerWindow.close();
|
|
775
|
-
if (mainWindow) mainWindow.show();
|
|
776
|
-
});
|
|
777
|
-
|
|
778
|
-
// =====================
|
|
779
|
-
// IPC Handlers — Proactive Assistant
|
|
780
|
-
// =====================
|
|
781
|
-
ipcMain.on('toggle-proactive', (event, isOn) => {
|
|
782
|
-
if (isOn) {
|
|
783
|
-
startProactiveLoop(); // reads interval from config automatically
|
|
784
|
-
} else {
|
|
785
|
-
stopProactiveLoop();
|
|
786
|
-
}
|
|
787
|
-
});
|
|
788
|
-
|
|
789
|
-
ipcMain.on('record-behavior', (event, contextDescription) => {
|
|
790
|
-
recordBehavior(contextDescription);
|
|
791
|
-
});
|
|
792
|
-
|
|
793
|
-
// Direct action executor for proactive Accept (no Gemini involved)
|
|
794
|
-
ipcMain.handle('execute-proactive-action', async (event, action) => {
|
|
795
|
-
if (!action || action.type === 'none') {
|
|
796
|
-
return { success: false, message: 'ไม่มี action ที่จะดำเนินการค่ะ' };
|
|
797
|
-
}
|
|
798
|
-
try {
|
|
799
|
-
const result = await executeAction(action);
|
|
800
|
-
// Build a friendly confirmation message
|
|
801
|
-
const messages = {
|
|
802
|
-
open_url: `เปิดเว็บไซต์ให้แล้วค่ะ 🌐`,
|
|
803
|
-
open_app: `เปิดแอป ${action.target} ให้แล้วค่ะ 🚀`,
|
|
804
|
-
search: `ค้นหา "${action.target}" ให้แล้วค่ะ 🔍`,
|
|
805
|
-
web_automation: result || 'ดำเนินการเสร็จแล้วค่ะ ✅',
|
|
806
|
-
create_folder: `สร้างโฟลเดอร์ "${action.target}" แล้วค่ะ 📁`,
|
|
807
|
-
clipboard_write: `คัดลอกข้อความแล้วค่ะ 📋`,
|
|
808
|
-
learn_file: result || `เรียนรู้เอกสารเรียบร้อยค่ะ 📚`,
|
|
809
|
-
};
|
|
810
|
-
return {
|
|
811
|
-
success: true,
|
|
812
|
-
message: messages[action.type] || 'ดำเนินการเสร็จแล้วค่ะ ✅'
|
|
813
|
-
};
|
|
814
|
-
} catch (err) {
|
|
815
|
-
console.error('[ProactiveAction] Error:', err);
|
|
816
|
-
return { success: false, message: `เกิดข้อผิดพลาด: ${err.message}` };
|
|
817
|
-
}
|
|
818
|
-
});
|
|
819
|
-
|
|
820
|
-
ipcMain.handle('capture-silent-screen', async () => {
|
|
821
|
-
try {
|
|
822
|
-
const primaryDisplay = screen.getPrimaryDisplay();
|
|
823
|
-
const { width, height } = primaryDisplay.size;
|
|
824
|
-
|
|
825
|
-
const sources = await desktopCapturer.getSources({
|
|
826
|
-
types: ['screen'],
|
|
827
|
-
thumbnailSize: { width, height }
|
|
828
|
-
});
|
|
829
|
-
|
|
830
|
-
const primarySource = sources[0];
|
|
831
|
-
if (primarySource && primarySource.thumbnail) {
|
|
832
|
-
return primarySource.thumbnail.toDataURL();
|
|
833
|
-
}
|
|
834
|
-
return null;
|
|
835
|
-
} catch (err) {
|
|
836
|
-
console.error("Error silently capturing screen:", err);
|
|
837
|
-
return null;
|
|
838
|
-
}
|
|
839
|
-
});
|
|
840
|
-
|
|
841
|
-
// =====================
|
|
842
|
-
// Action Executor
|
|
843
|
-
// =====================
|
|
844
|
-
async function executeAction(action) {
|
|
845
|
-
console.log("Executing action:", action);
|
|
846
|
-
|
|
847
|
-
switch (action.type) {
|
|
848
|
-
case 'open_url':
|
|
849
|
-
openWebsite(action.target);
|
|
850
|
-
break;
|
|
851
|
-
case 'search':
|
|
852
|
-
openSearch(action.target);
|
|
853
|
-
break;
|
|
854
|
-
case 'open_app':
|
|
855
|
-
openApp(action.target);
|
|
856
|
-
break;
|
|
857
|
-
case 'web_automation':
|
|
858
|
-
return await performWebAutomation(action.target);
|
|
859
|
-
case 'create_folder':
|
|
860
|
-
createFolder(action.target);
|
|
861
|
-
break;
|
|
862
|
-
case 'open_file':
|
|
863
|
-
const fileRes = await openFile(action.target);
|
|
864
|
-
return fileRes || `Successfully opened file: ${action.target} ✅`;
|
|
865
|
-
case 'open_folder':
|
|
866
|
-
const folderRes = await openFile(action.target);
|
|
867
|
-
return folderRes || `Successfully opened folder: ${action.target} ✅`;
|
|
868
|
-
case 'delete_file':
|
|
869
|
-
await deleteFile(action.target);
|
|
870
|
-
break;
|
|
871
|
-
case 'find_path': {
|
|
872
|
-
const result = findPath(action.target, {
|
|
873
|
-
type: action.pathType,
|
|
874
|
-
maxResults: 10
|
|
875
|
-
});
|
|
876
|
-
if (!result.success) {
|
|
877
|
-
return result.message;
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
if (action.openAfter === true) {
|
|
881
|
-
if (result.matches.length === 1) {
|
|
882
|
-
const match = result.matches[0];
|
|
883
|
-
const openResult = await openFile(match.path);
|
|
884
|
-
return openResult || `Successfully found and opened ${match.type === 'dir' ? 'folder' : 'file'}: ${match.path} ✅`;
|
|
885
|
-
}
|
|
886
|
-
return `Found multiple matches for "${action.target}". Please be more specific:\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
|
|
887
|
-
}
|
|
888
|
-
|
|
889
|
-
return `Found matches for "${action.target}":\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
|
|
890
|
-
}
|
|
891
|
-
case 'clipboard_write':
|
|
892
|
-
clipboard.writeText(action.target);
|
|
893
|
-
break;
|
|
894
|
-
case 'learn_file':
|
|
895
|
-
return await indexFile(action.target);
|
|
896
|
-
case 'learn_folder':
|
|
897
|
-
const { indexFolder } = require('./src/AI_Brain/knowledge_base');
|
|
898
|
-
return await indexFolder(action.target);
|
|
899
|
-
case 'mcp_tool':
|
|
900
|
-
const mcpResult = await mcpManager.callTool(action.server, action.target, action.args);
|
|
901
|
-
return JSON.stringify(mcpResult.content);
|
|
902
|
-
case 'mouse_move':
|
|
903
|
-
return await granularAutomation.mouseMove(action.x, action.y);
|
|
904
|
-
case 'mouse_click':
|
|
905
|
-
return await granularAutomation.mouseClick(action.x, action.y, action.button || 1);
|
|
906
|
-
case 'type_text':
|
|
907
|
-
return await granularAutomation.typeText(action.target);
|
|
908
|
-
case 'key_tap':
|
|
909
|
-
return await granularAutomation.keyTap(action.target);
|
|
910
|
-
case 'plugin':
|
|
911
|
-
return await pluginManager.executePlugin(action.pluginName, action.target);
|
|
912
|
-
case 'system_automation':
|
|
913
|
-
return await handleSystemAutomation(action.target);
|
|
914
|
-
}
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
async function handleSystemAutomation(target) {
|
|
918
|
-
const [cmd, value] = target.split(':');
|
|
919
|
-
switch (cmd) {
|
|
920
|
-
case 'volume':
|
|
921
|
-
return await SystemAutomation.setVolume(parseInt(value));
|
|
922
|
-
case 'mute':
|
|
923
|
-
return await SystemAutomation.mute();
|
|
924
|
-
case 'brightness':
|
|
925
|
-
return await SystemAutomation.setBrightness(parseInt(value));
|
|
926
|
-
case 'sleep':
|
|
927
|
-
return await SystemAutomation.sleep();
|
|
928
|
-
case 'restart':
|
|
929
|
-
return await SystemAutomation.restart();
|
|
930
|
-
case 'shutdown':
|
|
931
|
-
return await SystemAutomation.shutdown();
|
|
932
|
-
case 'minimize_all':
|
|
933
|
-
return await SystemAutomation.minimizeAll();
|
|
934
|
-
default:
|
|
935
|
-
// Handle cases where target is just the command (like 'sleep')
|
|
936
|
-
if (SystemAutomation[target]) {
|
|
937
|
-
return await SystemAutomation[target]();
|
|
938
|
-
}
|
|
939
|
-
throw new Error(`Unknown system automation command: ${target}`);
|
|
940
|
-
}
|
|
941
|
-
}
|