@pheem49/mint 1.4.2 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GUIDE_TH.md +113 -0
- package/README.md +267 -78
- package/assets/CLI_Screen.png +0 -0
- package/main.js +76 -890
- package/mint-cli-logic.js +3 -107
- package/mint-cli.js +594 -29
- package/models/Shiroko_Model/Shiroko/Shiroko_Core/72d86db84cfa9730b894c241fd24c0db.png +0 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core/items_pinned_to_model.json +14 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253.exp3.json +10 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253/347/234/274/347/217/240/346/221/207/346/231/203.exp3.json +15 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//345/233/264/350/243/231.exp3.json +10 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/215/347/205/247.exp3.json +50 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/277/347/254/224.exp3.json +10 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//347/202/271/344/270/200/344/270/213.exp3.json +10 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//347/214/253/345/222/252/346/273/244/351/225/234.exp3.json +10 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//347/234/274/351/225/234.exp3.json +10 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_00.png +0 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_01.png +0 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_02.png +0 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_03.png +0 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.cdi3.json +1498 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.moc3 +0 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.model3.json +47 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.physics3.json +6658 -0
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.vtube.json +1299 -0
- package/models/Shiroko_Model/Shiroko//342/232/241/351/253/230/344/272/256/342/232/241/344/275/277/347/224/250/346/225/231/347/250/213/344/270/216/346/263/250/346/204/217/344/272/213/351/241/271.txt +23 -0
- package/package.json +37 -4
- package/src/AI_Brain/Gemini_API.js +223 -65
- package/src/AI_Brain/autonomous_brain.js +11 -0
- package/src/AI_Brain/behavior_memory.js +26 -5
- package/src/AI_Brain/headless_agent.js +4 -0
- package/src/AI_Brain/knowledge_base.js +61 -8
- package/src/AI_Brain/memory_store.js +354 -10
- package/src/Automation_Layer/file_operations.js +1 -1
- package/src/CLI/chat_router.js +20 -7
- package/src/CLI/chat_ui.js +596 -825
- package/src/CLI/code_agent.js +347 -56
- package/src/CLI/gmail_auth.js +210 -0
- package/src/CLI/image_input.js +90 -0
- package/src/CLI/list_features.js +2 -0
- package/src/CLI/onboarding.js +364 -55
- package/src/CLI/updater.js +210 -0
- package/src/Channels/brave_search_bridge.js +35 -0
- package/src/Channels/discord_bridge.js +68 -0
- package/src/Channels/google_search_bridge.js +38 -0
- package/src/Channels/line_bridge.js +60 -0
- package/src/Channels/slack_bridge.js +53 -0
- package/src/Channels/telegram_bridge.js +49 -0
- package/src/Channels/whatsapp_bridge.js +55 -0
- package/src/Command_Parser/parser.js +12 -1
- package/src/Plugins/gmail.js +251 -0
- package/src/Plugins/google_calendar.js +245 -19
- package/src/Plugins/notion.js +256 -0
- package/src/System/action_executor.js +178 -0
- package/src/System/bridge_manager.js +76 -0
- package/src/System/chat_history_manager.js +23 -5
- package/src/System/config_manager.js +71 -7
- package/src/System/custom_workflows.js +31 -2
- package/src/System/google_tts_urls.js +51 -0
- package/src/System/granular_automation.js +122 -53
- package/src/System/ipc_handlers.js +238 -0
- package/src/System/proactive_loop.js +153 -0
- package/src/System/safety_manager.js +273 -0
- package/src/System/sandbox_runner.js +182 -0
- package/src/System/screen_capture.js +175 -0
- package/src/System/system_automation.js +127 -81
- package/src/System/system_info.js +70 -0
- package/src/System/task_manager.js +15 -5
- package/src/System/tool_registry.js +280 -0
- package/src/System/window_manager.js +212 -0
- package/src/UI/live2d_manager.js +368 -0
- package/src/UI/renderer.js +208 -24
- package/src/UI/settings.html +24 -0
- package/src/UI/settings.js +14 -4
- package/src/UI/styles.css +466 -32
- package/.codex +0 -0
- package/docs/assets/Agent_Mint.png +0 -0
- package/docs/assets/CLI_Screen.png +0 -0
- package/docs/assets/Settings.png +0 -0
- package/docs/assets/icon.png +0 -0
- package/docs/index.html +0 -132
- package/docs/style.css +0 -579
- package/index.html +0 -16
- package/src/UI/index.html +0 -126
- package/tech_news.txt +0 -3
- package/test_knowledge.txt +0 -3
- package/tests/agent_orchestrator.test.js +0 -41
- package/tests/chat_router.test.js +0 -42
- package/tests/code_agent.test.js +0 -69
- package/tests/config_manager.test.js +0 -141
- package/tests/docker.test.js +0 -46
- package/tests/file_operations.test.js +0 -57
- package/tests/memory_store.test.js +0 -185
- package/tests/provider_routing.test.js +0 -67
- package/tests/spotify.test.js +0 -201
- package/tests/system_monitor.test.js +0 -37
- package/tests/workspace_manager.test.js +0 -56
package/main.js
CHANGED
|
@@ -1,360 +1,96 @@
|
|
|
1
|
-
const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard
|
|
2
|
-
const path = require('path');
|
|
1
|
+
const { app, BrowserWindow, ipcMain, shell, globalShortcut, clipboard } = require('electron');
|
|
3
2
|
require('dotenv').config();
|
|
4
3
|
|
|
5
|
-
const { handleChat, resetChat, getChatTranscript, translateImageContent, refreshApiKeyFromConfig } = require('./src/AI_Brain/Gemini_API');
|
|
6
|
-
const { openApp } = require('./src/Automation_Layer/open_app');
|
|
7
|
-
const { openWebsite, openSearch } = require('./src/Automation_Layer/open_website');
|
|
8
|
-
const { performWebAutomation } = require('./src/Automation_Layer/browser_automation');
|
|
9
|
-
const { createFolder, openFile, deleteFile, findPath } = require('./src/Automation_Layer/file_operations');
|
|
10
4
|
const { getSystemInfo, getWeather } = require('./src/System/system_info');
|
|
11
5
|
const { readConfig, writeConfig } = require('./src/System/config_manager');
|
|
12
6
|
const { parseCommand } = require('./src/Command_Parser/parser');
|
|
13
|
-
const
|
|
14
|
-
const {
|
|
15
|
-
const {
|
|
16
|
-
const {
|
|
7
|
+
const { getGoogleTtsUrls } = require('./src/System/google_tts_urls');
|
|
8
|
+
const { createWindowManager } = require('./src/System/window_manager');
|
|
9
|
+
const { createProactiveLoop } = require('./src/System/proactive_loop');
|
|
10
|
+
const { createScreenCaptureController } = require('./src/System/screen_capture');
|
|
11
|
+
const { registerIpcHandlers } = require('./src/System/ipc_handlers');
|
|
17
12
|
|
|
18
|
-
const SystemAutomation = require('./src/System/system_automation');
|
|
19
13
|
const systemEvents = require('./src/System/system_events');
|
|
20
14
|
const customWorkflows = require('./src/System/custom_workflows');
|
|
21
15
|
const mcpManager = require('./src/Plugins/mcp_manager');
|
|
22
|
-
const granularAutomation = require('./src/System/granular_automation');
|
|
23
|
-
const googleTTS = require('google-tts-api');
|
|
24
16
|
|
|
25
|
-
let
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
let widgetWindow = null;
|
|
30
|
-
let proactiveGlowWindow = null;
|
|
31
|
-
let tray = null;
|
|
32
|
-
|
|
33
|
-
// =====================
|
|
34
|
-
// Proactive Loop
|
|
35
|
-
// =====================
|
|
36
|
-
let proactiveIntervalHandle = null;
|
|
37
|
-
|
|
38
|
-
async function runProactiveCycle() {
|
|
39
|
-
if (!mainWindow || mainWindow.isDestroyed()) return;
|
|
40
|
-
try {
|
|
41
|
-
// Show Glow
|
|
42
|
-
showProactiveGlow();
|
|
43
|
-
|
|
44
|
-
// Silent screen capture
|
|
45
|
-
const primaryDisplay = screen.getPrimaryDisplay();
|
|
46
|
-
// Downscale to 50% for performance
|
|
47
|
-
const width = Math.floor(primaryDisplay.size.width * 0.5);
|
|
48
|
-
const height = Math.floor(primaryDisplay.size.height * 0.5);
|
|
49
|
-
|
|
50
|
-
const sources = await desktopCapturer.getSources({
|
|
51
|
-
types: ['screen'],
|
|
52
|
-
thumbnailSize: { width, height }
|
|
53
|
-
});
|
|
54
|
-
const primarySource = sources[0];
|
|
55
|
-
if (!primarySource || !primarySource.thumbnail) return;
|
|
56
|
-
|
|
57
|
-
// Compress image to JPEG to save huge base64 payload size and memory
|
|
58
|
-
// .toJPEG(quality) where quality is 0-100
|
|
59
|
-
const base64Image = primarySource.thumbnail.toJPEG(60).toString('base64');
|
|
60
|
-
const behaviorSummary = getBehaviorSummary();
|
|
61
|
-
|
|
62
|
-
const result = await analyzeAndSuggest(base64Image, behaviorSummary);
|
|
63
|
-
|
|
64
|
-
if (result && result.message && Array.isArray(result.suggestions)) {
|
|
65
|
-
// Record the observed context into behavior memory
|
|
66
|
-
if (result.context) recordBehavior(result.context);
|
|
67
|
-
|
|
68
|
-
// Push suggestion to the renderer
|
|
69
|
-
if (mainWindow && !mainWindow.isDestroyed()) {
|
|
70
|
-
mainWindow.webContents.send('proactive-suggestion', result);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Hide Glow after analysis
|
|
75
|
-
hideProactiveGlow();
|
|
76
|
-
} catch (err) {
|
|
77
|
-
console.error('[Proactive] Cycle error:', err.message);
|
|
78
|
-
hideProactiveGlow();
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function createProactiveGlowWindow() {
|
|
83
|
-
if (proactiveGlowWindow) return;
|
|
84
|
-
const { width, height } = screen.getPrimaryDisplay().bounds;
|
|
85
|
-
|
|
86
|
-
proactiveGlowWindow = new BrowserWindow({
|
|
87
|
-
width, height,
|
|
88
|
-
x: 0, y: 0,
|
|
89
|
-
frame: false,
|
|
90
|
-
transparent: true,
|
|
91
|
-
alwaysOnTop: true,
|
|
92
|
-
skipTaskbar: true,
|
|
93
|
-
focusable: false,
|
|
94
|
-
show: false,
|
|
95
|
-
webPreferences: {
|
|
96
|
-
nodeIntegration: false,
|
|
97
|
-
contextIsolation: true
|
|
98
|
-
}
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
proactiveGlowWindow.setIgnoreMouseEvents(true);
|
|
102
|
-
proactiveGlowWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
|
|
103
|
-
proactiveGlowWindow.setAlwaysOnTop(true, 'screen-saver');
|
|
104
|
-
proactiveGlowWindow.loadFile('src/UI/proactive-glow.html');
|
|
105
|
-
|
|
106
|
-
proactiveGlowWindow.on('closed', () => { proactiveGlowWindow = null; });
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
function showProactiveGlow() {
|
|
110
|
-
if (!proactiveGlowWindow) createProactiveGlowWindow();
|
|
111
|
-
if (proactiveGlowWindow) proactiveGlowWindow.showInactive();
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function hideProactiveGlow() {
|
|
115
|
-
if (proactiveGlowWindow && !proactiveGlowWindow.isDestroyed()) {
|
|
116
|
-
proactiveGlowWindow.hide();
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
function startProactiveLoop(intervalSec) {
|
|
121
|
-
// Stop existing loop first
|
|
122
|
-
if (proactiveIntervalHandle) {
|
|
123
|
-
clearInterval(proactiveIntervalHandle);
|
|
124
|
-
proactiveIntervalHandle = null;
|
|
125
|
-
}
|
|
126
|
-
// Read from config if not provided
|
|
127
|
-
const cfg = readConfig();
|
|
128
|
-
const ms = (intervalSec || cfg.proactiveInterval || 60) * 1000;
|
|
129
|
-
console.log(`[Proactive] Starting loop — interval: ${ms / 1000}s`);
|
|
130
|
-
proactiveIntervalHandle = setInterval(runProactiveCycle, ms);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function stopProactiveLoop() {
|
|
134
|
-
if (proactiveIntervalHandle) {
|
|
135
|
-
clearInterval(proactiveIntervalHandle);
|
|
136
|
-
proactiveIntervalHandle = null;
|
|
137
|
-
console.log('[Proactive] Stopped proactive loop.');
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
// Check idle state every minute to pause/resume background tasks
|
|
142
|
-
const IDLE_THRESHOLD_SEC = 300; // 5 minutes
|
|
143
|
-
setInterval(() => {
|
|
144
|
-
if (!proactiveIntervalHandle) return; // Only manage if it's supposed to be active (Smart Context is ON)
|
|
145
|
-
|
|
146
|
-
// powerMonitor.getSystemIdleTime() is available after app is ready
|
|
147
|
-
if (app.isReady()) {
|
|
148
|
-
const idleSec = powerMonitor.getSystemIdleTime();
|
|
149
|
-
if (idleSec >= IDLE_THRESHOLD_SEC) {
|
|
150
|
-
console.log(`[System Idle] User idle for ${idleSec}s. Pausing Proactive loop to save resources.`);
|
|
151
|
-
stopProactiveLoop();
|
|
152
|
-
|
|
153
|
-
// Wait for user to come back
|
|
154
|
-
const resumeChecker = setInterval(() => {
|
|
155
|
-
if (powerMonitor.getSystemIdleTime() < 10) {
|
|
156
|
-
console.log('[System Idle] User returned. Resuming Proactive loop.');
|
|
157
|
-
clearInterval(resumeChecker);
|
|
158
|
-
startProactiveLoop();
|
|
159
|
-
}
|
|
160
|
-
}, 5000);
|
|
161
|
-
}
|
|
17
|
+
let geminiServices = null;
|
|
18
|
+
function getGeminiServices() {
|
|
19
|
+
if (!geminiServices) {
|
|
20
|
+
geminiServices = require('./src/AI_Brain/Gemini_API');
|
|
162
21
|
}
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
function createWindow() {
|
|
166
|
-
mainWindow = new BrowserWindow({
|
|
167
|
-
width: 600,
|
|
168
|
-
height: 800,
|
|
169
|
-
icon: path.join(__dirname, 'assets', 'icon.png'),
|
|
170
|
-
webPreferences: {
|
|
171
|
-
preload: path.join(__dirname, 'preload.js'),
|
|
172
|
-
nodeIntegration: false,
|
|
173
|
-
contextIsolation: true,
|
|
174
|
-
},
|
|
175
|
-
frame: false,
|
|
176
|
-
transparent: true,
|
|
177
|
-
resizable: true,
|
|
178
|
-
show: false
|
|
179
|
-
});
|
|
180
|
-
|
|
181
|
-
mainWindow.loadFile('src/UI/index.html');
|
|
182
|
-
|
|
183
|
-
mainWindow.on('ready-to-show', () => {
|
|
184
|
-
mainWindow.show();
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
mainWindow.on('close', function (event) {
|
|
188
|
-
if (!app.isQuiting) {
|
|
189
|
-
event.preventDefault();
|
|
190
|
-
mainWindow.hide();
|
|
191
|
-
}
|
|
192
|
-
return false;
|
|
193
|
-
});
|
|
194
|
-
|
|
195
|
-
mainWindow.on('focus', () => {
|
|
196
|
-
// clearFloatingUnread(); // Disabled
|
|
197
|
-
});
|
|
22
|
+
return geminiServices;
|
|
198
23
|
}
|
|
199
24
|
|
|
200
|
-
function
|
|
201
|
-
|
|
202
|
-
let icon = nativeImage.createFromPath(iconPath);
|
|
203
|
-
// Optional: resize it for the tray if needed, but Electron handles this natively on many OSs
|
|
204
|
-
icon = icon.resize({ width: 16, height: 16 });
|
|
205
|
-
|
|
206
|
-
tray = new Tray(icon);
|
|
207
|
-
tray.setToolTip('Mint AI Assistant');
|
|
208
|
-
|
|
209
|
-
const contextMenu = Menu.buildFromTemplate([
|
|
210
|
-
{ label: 'Show App', click: () => { if (mainWindow) mainWindow.show(); } },
|
|
211
|
-
{ label: 'Settings', click: () => { createSettingsWindow(); } },
|
|
212
|
-
{ type: 'separator' },
|
|
213
|
-
{ label: 'Quit', click: () => {
|
|
214
|
-
app.isQuiting = true;
|
|
215
|
-
app.quit();
|
|
216
|
-
}}
|
|
217
|
-
]);
|
|
218
|
-
|
|
219
|
-
tray.setContextMenu(contextMenu);
|
|
220
|
-
|
|
221
|
-
tray.on('click', () => {
|
|
222
|
-
if (mainWindow) {
|
|
223
|
-
if (mainWindow.isVisible()) {
|
|
224
|
-
mainWindow.hide();
|
|
225
|
-
} else {
|
|
226
|
-
mainWindow.show();
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
function createSettingsWindow() {
|
|
233
|
-
if (settingsWindow) {
|
|
234
|
-
settingsWindow.focus();
|
|
235
|
-
return;
|
|
236
|
-
}
|
|
237
|
-
settingsWindow = new BrowserWindow({
|
|
238
|
-
width: 720,
|
|
239
|
-
height: 620,
|
|
240
|
-
minWidth: 640,
|
|
241
|
-
minHeight: 560,
|
|
242
|
-
icon: path.join(__dirname, 'assets', 'icon.png'),
|
|
243
|
-
webPreferences: {
|
|
244
|
-
preload: path.join(__dirname, 'preload-settings.js'),
|
|
245
|
-
nodeIntegration: false,
|
|
246
|
-
contextIsolation: true,
|
|
247
|
-
},
|
|
248
|
-
frame: false,
|
|
249
|
-
transparent: true,
|
|
250
|
-
resizable: true,
|
|
251
|
-
parent: mainWindow,
|
|
252
|
-
});
|
|
253
|
-
settingsWindow.loadFile('src/UI/settings.html');
|
|
254
|
-
settingsWindow.on('closed', () => { settingsWindow = null; });
|
|
25
|
+
function getActionExecutor() {
|
|
26
|
+
return require('./src/System/action_executor');
|
|
255
27
|
}
|
|
256
28
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
29
|
+
const projectRoot = __dirname;
|
|
30
|
+
const windowManager = createWindowManager(projectRoot);
|
|
31
|
+
const proactiveLoop = createProactiveLoop({
|
|
32
|
+
app,
|
|
33
|
+
projectRoot,
|
|
34
|
+
readConfig,
|
|
35
|
+
getMainWindow: windowManager.getMainWindow
|
|
36
|
+
});
|
|
37
|
+
const screenCapture = createScreenCaptureController({
|
|
38
|
+
projectRoot,
|
|
39
|
+
translateImageContent: (...args) => getGeminiServices().translateImageContent(...args),
|
|
40
|
+
getMainWindow: windowManager.getMainWindow
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
registerIpcHandlers({
|
|
44
|
+
app,
|
|
45
|
+
ipcMain,
|
|
46
|
+
shell,
|
|
47
|
+
clipboard,
|
|
48
|
+
windowManager,
|
|
49
|
+
proactiveLoop,
|
|
50
|
+
screenCapture,
|
|
51
|
+
services: {
|
|
52
|
+
handleChat: (...args) => getGeminiServices().handleChat(...args),
|
|
53
|
+
resetChat: (...args) => getGeminiServices().resetChat(...args),
|
|
54
|
+
getChatTranscript: (...args) => getGeminiServices().getChatTranscript(...args),
|
|
55
|
+
refreshApiKeyFromConfig: (...args) => getGeminiServices().refreshApiKeyFromConfig(...args),
|
|
56
|
+
getSystemInfo,
|
|
57
|
+
getWeather,
|
|
58
|
+
readConfig,
|
|
59
|
+
writeConfig,
|
|
60
|
+
parseCommand,
|
|
61
|
+
executeAction: (...args) => getActionExecutor().executeAction(...args),
|
|
62
|
+
getGoogleTtsUrls,
|
|
63
|
+
customWorkflows
|
|
261
64
|
}
|
|
65
|
+
});
|
|
262
66
|
|
|
263
|
-
|
|
264
|
-
const
|
|
265
|
-
const
|
|
266
|
-
|
|
267
|
-
spotlightWindow = new BrowserWindow({
|
|
268
|
-
width: windowWidth,
|
|
269
|
-
height: windowHeight,
|
|
270
|
-
x: Math.floor((screenWidth - windowWidth) / 2),
|
|
271
|
-
y: Math.floor(screenHeight * 0.25), // 25% from top
|
|
272
|
-
frame: false,
|
|
273
|
-
transparent: true,
|
|
274
|
-
alwaysOnTop: true,
|
|
275
|
-
skipTaskbar: true,
|
|
276
|
-
show: false,
|
|
277
|
-
webPreferences: {
|
|
278
|
-
preload: path.join(__dirname, 'src/UI/preload-spotlight.js'),
|
|
279
|
-
nodeIntegration: false,
|
|
280
|
-
contextIsolation: true,
|
|
281
|
-
}
|
|
282
|
-
});
|
|
283
|
-
|
|
284
|
-
spotlightWindow.loadFile('src/UI/spotlight.html');
|
|
285
|
-
|
|
286
|
-
spotlightWindow.on('blur', () => {
|
|
287
|
-
spotlightWindow.hide();
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
spotlightWindow.on('closed', () => {
|
|
291
|
-
spotlightWindow = null;
|
|
292
|
-
});
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
// Floating window logic removed
|
|
67
|
+
app.whenReady().then(() => {
|
|
68
|
+
const config = readConfig();
|
|
69
|
+
const mainWindow = windowManager.createMainWindow();
|
|
70
|
+
windowManager.createTray();
|
|
296
71
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
widgetWindow = new BrowserWindow({
|
|
301
|
-
width: 150,
|
|
302
|
-
height: 150,
|
|
303
|
-
frame: false,
|
|
304
|
-
transparent: true,
|
|
305
|
-
resizable: false,
|
|
306
|
-
alwaysOnTop: true,
|
|
307
|
-
skipTaskbar: true,
|
|
308
|
-
show: true,
|
|
309
|
-
webPreferences: {
|
|
310
|
-
preload: require('path').join(__dirname, 'src/UI/preload-widget.js'),
|
|
311
|
-
nodeIntegration: false,
|
|
312
|
-
contextIsolation: true
|
|
72
|
+
mainWindow.once('ready-to-show', () => {
|
|
73
|
+
if (config.showDesktopWidget !== false) {
|
|
74
|
+
setTimeout(() => windowManager.createWidgetWindow(), 300);
|
|
313
75
|
}
|
|
314
|
-
});
|
|
315
|
-
|
|
316
|
-
widgetWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
|
|
317
|
-
widgetWindow.setAlwaysOnTop(true, 'floating');
|
|
318
|
-
|
|
319
|
-
// Position Top-Right loosely
|
|
320
|
-
try {
|
|
321
|
-
const primaryDisplay = require('electron').screen.getPrimaryDisplay();
|
|
322
|
-
const { width, height, x, y } = primaryDisplay.workArea;
|
|
323
|
-
widgetWindow.setPosition(x + width - 150 - 40, y + 40);
|
|
324
|
-
} catch (e) {}
|
|
325
76
|
|
|
326
|
-
|
|
77
|
+
setTimeout(() => {
|
|
78
|
+
mcpManager.init().catch(err => console.error('[MCP] Init Error:', err));
|
|
327
79
|
|
|
328
|
-
|
|
329
|
-
|
|
80
|
+
const bridgeManager = require('./src/System/bridge_manager');
|
|
81
|
+
bridgeManager.init().catch(err => console.error('[BridgeManager] Init Error:', err));
|
|
82
|
+
}, 1000);
|
|
330
83
|
});
|
|
331
|
-
}
|
|
332
84
|
|
|
333
|
-
// Floating unread logic removed
|
|
334
|
-
|
|
335
|
-
app.whenReady().then(() => {
|
|
336
|
-
const config = readConfig();
|
|
337
|
-
createWindow();
|
|
338
|
-
createTray();
|
|
339
|
-
// createFloatingWindow(); // Removed
|
|
340
|
-
|
|
341
|
-
// Only show AI widget if enabled in settings
|
|
342
|
-
if (config.showDesktopWidget !== false) {
|
|
343
|
-
createWidgetWindow();
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
// Initialize MCP Servers
|
|
347
|
-
mcpManager.init().catch(err => console.error('[MCP] Init Error:', err));
|
|
348
|
-
|
|
349
|
-
// Start monitoring system events (battery, wifi, etc.)
|
|
350
85
|
systemEvents.startMonitoring();
|
|
351
86
|
if (config.enableCustomWorkflows !== false) {
|
|
352
87
|
customWorkflows.startMonitoring(mainWindow.webContents);
|
|
353
88
|
}
|
|
354
89
|
|
|
355
90
|
systemEvents.on('low-battery', (level) => {
|
|
356
|
-
|
|
357
|
-
|
|
91
|
+
const currentMainWindow = windowManager.getMainWindow();
|
|
92
|
+
if (currentMainWindow && !currentMainWindow.isDestroyed()) {
|
|
93
|
+
currentMainWindow.webContents.send('proactive-notification', {
|
|
358
94
|
message: `⚠️ Battery is low (${level}%). Please plug in your charger. ✨`,
|
|
359
95
|
type: 'warning'
|
|
360
96
|
});
|
|
@@ -362,580 +98,30 @@ app.whenReady().then(() => {
|
|
|
362
98
|
});
|
|
363
99
|
|
|
364
100
|
systemEvents.on('connection-change', (isOnline) => {
|
|
365
|
-
|
|
101
|
+
const currentMainWindow = windowManager.getMainWindow();
|
|
102
|
+
if (currentMainWindow && !currentMainWindow.isDestroyed()) {
|
|
366
103
|
const msg = isOnline ? '✅ Internet connection restored. ✨' : '❌ Internet connection lost.';
|
|
367
|
-
|
|
104
|
+
currentMainWindow.webContents.send('proactive-notification', { message: msg, type: 'info' });
|
|
368
105
|
}
|
|
369
106
|
});
|
|
370
107
|
|
|
371
|
-
globalShortcut.register('CommandOrControl+Shift+Space',
|
|
372
|
-
|
|
373
|
-
if (mainWindow.isVisible()) {
|
|
374
|
-
mainWindow.hide();
|
|
375
|
-
} else {
|
|
376
|
-
mainWindow.show();
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
});
|
|
108
|
+
globalShortcut.register('CommandOrControl+Shift+Space', windowManager.toggleMainWindow);
|
|
109
|
+
globalShortcut.register('Alt+Space', windowManager.toggleSpotlightWindow);
|
|
380
110
|
|
|
381
|
-
|
|
382
|
-
if (spotlightWindow && spotlightWindow.isVisible()) {
|
|
383
|
-
spotlightWindow.hide();
|
|
384
|
-
} else {
|
|
385
|
-
createSpotlightWindow();
|
|
386
|
-
spotlightWindow.show();
|
|
387
|
-
}
|
|
388
|
-
});
|
|
111
|
+
proactiveLoop.startIdleWatcher();
|
|
389
112
|
|
|
390
|
-
app.on('activate',
|
|
391
|
-
if (BrowserWindow.getAllWindows().length === 0)
|
|
113
|
+
app.on('activate', () => {
|
|
114
|
+
if (BrowserWindow.getAllWindows().length === 0) {
|
|
115
|
+
windowManager.createMainWindow();
|
|
116
|
+
}
|
|
392
117
|
});
|
|
393
118
|
});
|
|
394
119
|
|
|
395
|
-
app.on('window-all-closed', (
|
|
396
|
-
|
|
120
|
+
app.on('window-all-closed', (event) => {
|
|
121
|
+
event.preventDefault();
|
|
397
122
|
});
|
|
398
123
|
|
|
399
124
|
app.on('will-quit', () => {
|
|
400
125
|
globalShortcut.unregisterAll();
|
|
401
126
|
mcpManager.shutdown();
|
|
402
127
|
});
|
|
403
|
-
|
|
404
|
-
// =====================
|
|
405
|
-
// IPC Handlers — Chat
|
|
406
|
-
// =====================
|
|
407
|
-
ipcMain.handle('chat-message', async (event, message, base64Image = null, base64Audio = null) => {
|
|
408
|
-
try {
|
|
409
|
-
const rawResponse = await handleChat(message, base64Image, base64Audio);
|
|
410
|
-
const aiResponse = parseCommand(rawResponse);
|
|
411
|
-
|
|
412
|
-
if (aiResponse.action && aiResponse.action.type !== 'none') {
|
|
413
|
-
try {
|
|
414
|
-
const actionResult = await executeAction(aiResponse.action);
|
|
415
|
-
// If the action returned a string result (e.g. from Web Automation), append it.
|
|
416
|
-
if (actionResult && typeof actionResult === 'string') {
|
|
417
|
-
aiResponse.response += `\n\n${actionResult}`;
|
|
418
|
-
}
|
|
419
|
-
} catch (err) {
|
|
420
|
-
console.error("Action execution error:", err);
|
|
421
|
-
aiResponse.response += "\n\n(Note: I tried to execute the action, but an error occurred.)";
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
return aiResponse;
|
|
426
|
-
} catch (error) {
|
|
427
|
-
console.error('Chat error:', error);
|
|
428
|
-
return { response: 'Error communicating with Gemini API. Check your console and API key.', action: { type: 'none' } };
|
|
429
|
-
}
|
|
430
|
-
});
|
|
431
|
-
|
|
432
|
-
ipcMain.on('close-window', (event) => {
|
|
433
|
-
if (mainWindow) mainWindow.hide();
|
|
434
|
-
});
|
|
435
|
-
|
|
436
|
-
ipcMain.on('minimize-window', (event) => {
|
|
437
|
-
if (mainWindow) mainWindow.minimize();
|
|
438
|
-
});
|
|
439
|
-
|
|
440
|
-
ipcMain.on('quit-app', (event) => {
|
|
441
|
-
app.isQuiting = true;
|
|
442
|
-
app.quit();
|
|
443
|
-
});
|
|
444
|
-
|
|
445
|
-
ipcMain.on('maximize-window', (event) => {
|
|
446
|
-
if (mainWindow) {
|
|
447
|
-
if (mainWindow.isMaximized()) {
|
|
448
|
-
mainWindow.unmaximize();
|
|
449
|
-
} else {
|
|
450
|
-
mainWindow.maximize();
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
});
|
|
454
|
-
|
|
455
|
-
ipcMain.handle('reset-chat', () => {
|
|
456
|
-
resetChat();
|
|
457
|
-
return { success: true };
|
|
458
|
-
});
|
|
459
|
-
|
|
460
|
-
ipcMain.handle('get-chat-history', () => {
|
|
461
|
-
return getChatTranscript();
|
|
462
|
-
});
|
|
463
|
-
|
|
464
|
-
// =====================
|
|
465
|
-
// IPC Handlers — Settings
|
|
466
|
-
// =====================
|
|
467
|
-
ipcMain.handle('open-settings', () => {
|
|
468
|
-
createSettingsWindow();
|
|
469
|
-
});
|
|
470
|
-
|
|
471
|
-
ipcMain.handle('get-settings', () => {
|
|
472
|
-
return readConfig();
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
ipcMain.handle('save-settings', (event, config) => {
|
|
476
|
-
console.log('[Settings] Saving new config. MCP Servers count:', Object.keys(config.mcpServers || {}).length);
|
|
477
|
-
const result = writeConfig(config);
|
|
478
|
-
// Refresh API key if user updated it in settings
|
|
479
|
-
refreshApiKeyFromConfig();
|
|
480
|
-
// 🔔 Notify main chat window immediately
|
|
481
|
-
if (mainWindow && !mainWindow.isDestroyed()) {
|
|
482
|
-
mainWindow.webContents.send('settings-changed', config);
|
|
483
|
-
}
|
|
484
|
-
// Restart proactive loop with new interval if it's running
|
|
485
|
-
if (proactiveIntervalHandle) {
|
|
486
|
-
startProactiveLoop(config.proactiveInterval);
|
|
487
|
-
}
|
|
488
|
-
if (config.enableCustomWorkflows !== false) {
|
|
489
|
-
customWorkflows.startMonitoring(mainWindow.webContents);
|
|
490
|
-
} else {
|
|
491
|
-
customWorkflows.stopMonitoring();
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
// Toggle Desktop Widget
|
|
495
|
-
if (config.showDesktopWidget === false) {
|
|
496
|
-
if (widgetWindow && !widgetWindow.isDestroyed()) {
|
|
497
|
-
widgetWindow.close();
|
|
498
|
-
widgetWindow = null;
|
|
499
|
-
}
|
|
500
|
-
} else {
|
|
501
|
-
if (!widgetWindow || widgetWindow.isDestroyed()) {
|
|
502
|
-
createWidgetWindow();
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
return result;
|
|
507
|
-
});
|
|
508
|
-
|
|
509
|
-
ipcMain.on('set-ai-state', (event, state) => {
|
|
510
|
-
if (widgetWindow && !widgetWindow.isDestroyed()) {
|
|
511
|
-
widgetWindow.webContents.send('widget-state', state);
|
|
512
|
-
}
|
|
513
|
-
});
|
|
514
|
-
|
|
515
|
-
ipcMain.on('close-settings', () => {
|
|
516
|
-
if (settingsWindow) settingsWindow.close();
|
|
517
|
-
});
|
|
518
|
-
|
|
519
|
-
ipcMain.on('quit-app', () => {
|
|
520
|
-
app.isQuiting = true;
|
|
521
|
-
app.quit();
|
|
522
|
-
});
|
|
523
|
-
|
|
524
|
-
ipcMain.handle('open-custom-workflows', () => {
|
|
525
|
-
customWorkflows.openConfigFile();
|
|
526
|
-
});
|
|
527
|
-
|
|
528
|
-
ipcMain.handle('reload-custom-workflows', () => {
|
|
529
|
-
customWorkflows.loadWorkflows();
|
|
530
|
-
return { success: true };
|
|
531
|
-
});
|
|
532
|
-
|
|
533
|
-
// =====================
|
|
534
|
-
// IPC Handlers — Spotlight
|
|
535
|
-
// =====================
|
|
536
|
-
ipcMain.on('spotlight-close', () => {
|
|
537
|
-
if (spotlightWindow) spotlightWindow.close();
|
|
538
|
-
});
|
|
539
|
-
|
|
540
|
-
ipcMain.on('spotlight-hide', () => {
|
|
541
|
-
if (spotlightWindow) spotlightWindow.hide();
|
|
542
|
-
});
|
|
543
|
-
|
|
544
|
-
ipcMain.on('spotlight-submit', async (event, query) => {
|
|
545
|
-
console.log('[Spotlight] Submit:', query);
|
|
546
|
-
if (spotlightWindow) spotlightWindow.hide();
|
|
547
|
-
|
|
548
|
-
// Show main window and send the message
|
|
549
|
-
if (mainWindow) {
|
|
550
|
-
mainWindow.show();
|
|
551
|
-
// We can either just show it or actually trigger the chat
|
|
552
|
-
mainWindow.webContents.send('spotlight-to-chat', query);
|
|
553
|
-
}
|
|
554
|
-
});
|
|
555
|
-
|
|
556
|
-
ipcMain.on('spotlight-resize', (event, width, height) => {
|
|
557
|
-
if (spotlightWindow) {
|
|
558
|
-
spotlightWindow.setSize(width, height);
|
|
559
|
-
}
|
|
560
|
-
});
|
|
561
|
-
|
|
562
|
-
// Floating IPC removed
|
|
563
|
-
|
|
564
|
-
ipcMain.handle('open-external', (event, url) => {
|
|
565
|
-
shell.openExternal(url);
|
|
566
|
-
});
|
|
567
|
-
|
|
568
|
-
// =====================
|
|
569
|
-
// IPC Handlers — Clipboard
|
|
570
|
-
// =====================
|
|
571
|
-
ipcMain.handle('clipboard-read', () => {
|
|
572
|
-
return clipboard.readText();
|
|
573
|
-
});
|
|
574
|
-
|
|
575
|
-
ipcMain.handle('clipboard-write', (event, text) => {
|
|
576
|
-
clipboard.writeText(text);
|
|
577
|
-
return { success: true };
|
|
578
|
-
});
|
|
579
|
-
|
|
580
|
-
// =====================
|
|
581
|
-
// IPC Handlers — TTS
|
|
582
|
-
// =====================
|
|
583
|
-
ipcMain.handle('get-tts-urls', async (event, text) => {
|
|
584
|
-
try {
|
|
585
|
-
const isThai = /[\u0E00-\u0E7F]/.test(text);
|
|
586
|
-
const lang = isThai ? 'th' : 'en';
|
|
587
|
-
|
|
588
|
-
const results = googleTTS.getAllAudioUrls(text, {
|
|
589
|
-
lang: lang,
|
|
590
|
-
slow: false,
|
|
591
|
-
host: 'https://translate.google.com',
|
|
592
|
-
splitPunct: ',.?!;:',
|
|
593
|
-
});
|
|
594
|
-
return results;
|
|
595
|
-
} catch (e) {
|
|
596
|
-
console.error("TTS Error:", e);
|
|
597
|
-
return [];
|
|
598
|
-
}
|
|
599
|
-
});
|
|
600
|
-
|
|
601
|
-
// =====================
|
|
602
|
-
// IPC Handlers — System Info
|
|
603
|
-
// =====================
|
|
604
|
-
ipcMain.handle('get-system-info', async () => {
|
|
605
|
-
return getSystemInfo();
|
|
606
|
-
});
|
|
607
|
-
|
|
608
|
-
ipcMain.handle('get-weather', async (event, city) => {
|
|
609
|
-
return getWeather(city);
|
|
610
|
-
});
|
|
611
|
-
|
|
612
|
-
// =====================
|
|
613
|
-
// IPC Handlers — Screen Vision
|
|
614
|
-
// =====================
|
|
615
|
-
ipcMain.handle('start-screen-capture', async () => {
|
|
616
|
-
if (screenPickerWindow) return; // Prevent multiple windows
|
|
617
|
-
|
|
618
|
-
try {
|
|
619
|
-
// Capture primary display
|
|
620
|
-
const primaryDisplay = screen.getPrimaryDisplay();
|
|
621
|
-
const { width, height } = primaryDisplay.size;
|
|
622
|
-
|
|
623
|
-
const sources = await desktopCapturer.getSources({
|
|
624
|
-
types: ['screen'],
|
|
625
|
-
thumbnailSize: { width, height }
|
|
626
|
-
});
|
|
627
|
-
|
|
628
|
-
// Find the full primary screen (usually "Screen 1" or "Entire screen")
|
|
629
|
-
const primarySource = sources[0]; // Assuming single/primary monitor is the first
|
|
630
|
-
|
|
631
|
-
// Create transparent, borderless screen picker window
|
|
632
|
-
screenPickerWindow = new BrowserWindow({
|
|
633
|
-
width, height,
|
|
634
|
-
x: primaryDisplay.bounds.x, y: primaryDisplay.bounds.y,
|
|
635
|
-
fullscreen: true,
|
|
636
|
-
transparent: true,
|
|
637
|
-
frame: false,
|
|
638
|
-
alwaysOnTop: true,
|
|
639
|
-
skipTaskbar: true,
|
|
640
|
-
webPreferences: {
|
|
641
|
-
preload: path.join(__dirname, 'preload-picker.js'),
|
|
642
|
-
nodeIntegration: false,
|
|
643
|
-
contextIsolation: true
|
|
644
|
-
}
|
|
645
|
-
});
|
|
646
|
-
|
|
647
|
-
await screenPickerWindow.loadFile('src/UI/screenPicker.html');
|
|
648
|
-
|
|
649
|
-
// Ensure image data isn't missing
|
|
650
|
-
if(primarySource && primarySource.thumbnail) {
|
|
651
|
-
screenPickerWindow.webContents.send('screenshot-data', primarySource.thumbnail.toDataURL());
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
screenPickerWindow.on('closed', () => { screenPickerWindow = null; });
|
|
655
|
-
} catch (err) {
|
|
656
|
-
console.error("Error starting screen capture:", err);
|
|
657
|
-
}
|
|
658
|
-
});
|
|
659
|
-
|
|
660
|
-
// Received selection from the picker
|
|
661
|
-
ipcMain.on('vision-selection', (event, base64Image) => {
|
|
662
|
-
if (screenPickerWindow) screenPickerWindow.close();
|
|
663
|
-
|
|
664
|
-
// Relay image back to main window interface
|
|
665
|
-
if (mainWindow) {
|
|
666
|
-
mainWindow.webContents.send('vision-ready', base64Image);
|
|
667
|
-
mainWindow.show(); // Bring chat back to focus
|
|
668
|
-
}
|
|
669
|
-
});
|
|
670
|
-
|
|
671
|
-
let translateIntervalHandle = null;
|
|
672
|
-
let isTranslateRequestInFlight = false;
|
|
673
|
-
let translateCooldownUntil = 0;
|
|
674
|
-
const TRANSLATE_REFRESH_MS = 3000;
|
|
675
|
-
const TRANSLATE_FAILURE_COOLDOWN_MS = 15000;
|
|
676
|
-
|
|
677
|
-
// Start Continuous Live Translate
|
|
678
|
-
ipcMain.on('vision-translate-start', async (event, rect) => {
|
|
679
|
-
if (!screenPickerWindow) return;
|
|
680
|
-
|
|
681
|
-
screenPickerWindow.setIgnoreMouseEvents(true, { forward: true });
|
|
682
|
-
isTranslateRequestInFlight = false;
|
|
683
|
-
translateCooldownUntil = 0;
|
|
684
|
-
|
|
685
|
-
// Stop any existing loop
|
|
686
|
-
if (translateIntervalHandle) clearInterval(translateIntervalHandle);
|
|
687
|
-
|
|
688
|
-
// Initial capture immediately
|
|
689
|
-
captureAndTranslate(rect);
|
|
690
|
-
|
|
691
|
-
// Then start ticking every 3 seconds
|
|
692
|
-
translateIntervalHandle = setInterval(() => {
|
|
693
|
-
captureAndTranslate(rect);
|
|
694
|
-
}, TRANSLATE_REFRESH_MS);
|
|
695
|
-
});
|
|
696
|
-
|
|
697
|
-
// Stop Continuous Live Translate
|
|
698
|
-
ipcMain.on('vision-translate-stop', () => {
|
|
699
|
-
if (translateIntervalHandle) {
|
|
700
|
-
clearInterval(translateIntervalHandle);
|
|
701
|
-
translateIntervalHandle = null;
|
|
702
|
-
}
|
|
703
|
-
if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
|
|
704
|
-
screenPickerWindow.setIgnoreMouseEvents(false);
|
|
705
|
-
}
|
|
706
|
-
isTranslateRequestInFlight = false;
|
|
707
|
-
translateCooldownUntil = 0;
|
|
708
|
-
});
|
|
709
|
-
|
|
710
|
-
ipcMain.on('vision-overlay-interactable', (event, isInteractable) => {
|
|
711
|
-
if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
|
|
712
|
-
screenPickerWindow.setIgnoreMouseEvents(!isInteractable, { forward: true });
|
|
713
|
-
});
|
|
714
|
-
|
|
715
|
-
async function captureAndTranslate(rect) {
|
|
716
|
-
if (!screenPickerWindow || screenPickerWindow.isDestroyed()) return;
|
|
717
|
-
if (isTranslateRequestInFlight) return;
|
|
718
|
-
if (Date.now() < translateCooldownUntil) return;
|
|
719
|
-
|
|
720
|
-
try {
|
|
721
|
-
isTranslateRequestInFlight = true;
|
|
722
|
-
const primaryDisplay = screen.getPrimaryDisplay();
|
|
723
|
-
const sources = await desktopCapturer.getSources({
|
|
724
|
-
types: ['screen'],
|
|
725
|
-
thumbnailSize: {
|
|
726
|
-
width: primaryDisplay.size.width,
|
|
727
|
-
height: primaryDisplay.size.height
|
|
728
|
-
}
|
|
729
|
-
});
|
|
730
|
-
|
|
731
|
-
if (sources.length > 0) {
|
|
732
|
-
const screenImage = sources[0].thumbnail;
|
|
733
|
-
|
|
734
|
-
// Crop the specific rect out of the full screen image
|
|
735
|
-
// We use standard scale (assuming 1:1 scale for exact rect, although hidpi might vary.
|
|
736
|
-
// In a robust implementation we might need `Math.floor(rect.x * display.scaleFactor)`)
|
|
737
|
-
const croppedImage = screenImage.crop({
|
|
738
|
-
x: Math.round(rect.x),
|
|
739
|
-
y: Math.round(rect.y),
|
|
740
|
-
width: Math.round(rect.width),
|
|
741
|
-
height: Math.round(rect.height)
|
|
742
|
-
});
|
|
743
|
-
|
|
744
|
-
// Convert to JPEG Base64 for faster processing
|
|
745
|
-
const base64Crop = croppedImage.toJPEG(70).toString('base64');
|
|
746
|
-
const dataUri = `data:image/jpeg;base64,${base64Crop}`;
|
|
747
|
-
|
|
748
|
-
const translationResult = await translateImageContent(dataUri);
|
|
749
|
-
if (translationResult.retryableFailure) {
|
|
750
|
-
translateCooldownUntil = Date.now() + TRANSLATE_FAILURE_COOLDOWN_MS;
|
|
751
|
-
console.warn(`Live translation cooldown active for ${TRANSLATE_FAILURE_COOLDOWN_MS / 1000}s after retryable API failure.`);
|
|
752
|
-
} else {
|
|
753
|
-
translateCooldownUntil = 0;
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
if (screenPickerWindow && !screenPickerWindow.isDestroyed()) {
|
|
757
|
-
screenPickerWindow.webContents.send('vision-translate-result', translationResult.text);
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
} catch (err) {
|
|
761
|
-
console.error("Continuous translation loop failed:", err);
|
|
762
|
-
} finally {
|
|
763
|
-
isTranslateRequestInFlight = false;
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
ipcMain.on('vision-cancel', () => {
|
|
768
|
-
if (translateIntervalHandle) {
|
|
769
|
-
clearInterval(translateIntervalHandle);
|
|
770
|
-
translateIntervalHandle = null;
|
|
771
|
-
}
|
|
772
|
-
isTranslateRequestInFlight = false;
|
|
773
|
-
translateCooldownUntil = 0;
|
|
774
|
-
if (screenPickerWindow) screenPickerWindow.close();
|
|
775
|
-
if (mainWindow) mainWindow.show();
|
|
776
|
-
});
|
|
777
|
-
|
|
778
|
-
// =====================
|
|
779
|
-
// IPC Handlers — Proactive Assistant
|
|
780
|
-
// =====================
|
|
781
|
-
ipcMain.on('toggle-proactive', (event, isOn) => {
|
|
782
|
-
if (isOn) {
|
|
783
|
-
startProactiveLoop(); // reads interval from config automatically
|
|
784
|
-
} else {
|
|
785
|
-
stopProactiveLoop();
|
|
786
|
-
}
|
|
787
|
-
});
|
|
788
|
-
|
|
789
|
-
ipcMain.on('record-behavior', (event, contextDescription) => {
|
|
790
|
-
recordBehavior(contextDescription);
|
|
791
|
-
});
|
|
792
|
-
|
|
793
|
-
// Direct action executor for proactive Accept (no Gemini involved)
|
|
794
|
-
ipcMain.handle('execute-proactive-action', async (event, action) => {
|
|
795
|
-
if (!action || action.type === 'none') {
|
|
796
|
-
return { success: false, message: 'ไม่มี action ที่จะดำเนินการค่ะ' };
|
|
797
|
-
}
|
|
798
|
-
try {
|
|
799
|
-
const result = await executeAction(action);
|
|
800
|
-
// Build a friendly confirmation message
|
|
801
|
-
const messages = {
|
|
802
|
-
open_url: `เปิดเว็บไซต์ให้แล้วค่ะ 🌐`,
|
|
803
|
-
open_app: `เปิดแอป ${action.target} ให้แล้วค่ะ 🚀`,
|
|
804
|
-
search: `ค้นหา "${action.target}" ให้แล้วค่ะ 🔍`,
|
|
805
|
-
web_automation: result || 'ดำเนินการเสร็จแล้วค่ะ ✅',
|
|
806
|
-
create_folder: `สร้างโฟลเดอร์ "${action.target}" แล้วค่ะ 📁`,
|
|
807
|
-
clipboard_write: `คัดลอกข้อความแล้วค่ะ 📋`,
|
|
808
|
-
learn_file: result || `เรียนรู้เอกสารเรียบร้อยค่ะ 📚`,
|
|
809
|
-
};
|
|
810
|
-
return {
|
|
811
|
-
success: true,
|
|
812
|
-
message: messages[action.type] || 'ดำเนินการเสร็จแล้วค่ะ ✅'
|
|
813
|
-
};
|
|
814
|
-
} catch (err) {
|
|
815
|
-
console.error('[ProactiveAction] Error:', err);
|
|
816
|
-
return { success: false, message: `เกิดข้อผิดพลาด: ${err.message}` };
|
|
817
|
-
}
|
|
818
|
-
});
|
|
819
|
-
|
|
820
|
-
ipcMain.handle('capture-silent-screen', async () => {
|
|
821
|
-
try {
|
|
822
|
-
const primaryDisplay = screen.getPrimaryDisplay();
|
|
823
|
-
const { width, height } = primaryDisplay.size;
|
|
824
|
-
|
|
825
|
-
const sources = await desktopCapturer.getSources({
|
|
826
|
-
types: ['screen'],
|
|
827
|
-
thumbnailSize: { width, height }
|
|
828
|
-
});
|
|
829
|
-
|
|
830
|
-
const primarySource = sources[0];
|
|
831
|
-
if (primarySource && primarySource.thumbnail) {
|
|
832
|
-
return primarySource.thumbnail.toDataURL();
|
|
833
|
-
}
|
|
834
|
-
return null;
|
|
835
|
-
} catch (err) {
|
|
836
|
-
console.error("Error silently capturing screen:", err);
|
|
837
|
-
return null;
|
|
838
|
-
}
|
|
839
|
-
});
|
|
840
|
-
|
|
841
|
-
// =====================
|
|
842
|
-
// Action Executor
|
|
843
|
-
// =====================
|
|
844
|
-
async function executeAction(action) {
|
|
845
|
-
console.log("Executing action:", action);
|
|
846
|
-
|
|
847
|
-
switch (action.type) {
|
|
848
|
-
case 'open_url':
|
|
849
|
-
openWebsite(action.target);
|
|
850
|
-
break;
|
|
851
|
-
case 'search':
|
|
852
|
-
openSearch(action.target);
|
|
853
|
-
break;
|
|
854
|
-
case 'open_app':
|
|
855
|
-
openApp(action.target);
|
|
856
|
-
break;
|
|
857
|
-
case 'web_automation':
|
|
858
|
-
return await performWebAutomation(action.target);
|
|
859
|
-
case 'create_folder':
|
|
860
|
-
createFolder(action.target);
|
|
861
|
-
break;
|
|
862
|
-
case 'open_file':
|
|
863
|
-
const fileRes = await openFile(action.target);
|
|
864
|
-
return fileRes || `Successfully opened file: ${action.target} ✅`;
|
|
865
|
-
case 'open_folder':
|
|
866
|
-
const folderRes = await openFile(action.target);
|
|
867
|
-
return folderRes || `Successfully opened folder: ${action.target} ✅`;
|
|
868
|
-
case 'delete_file':
|
|
869
|
-
await deleteFile(action.target);
|
|
870
|
-
break;
|
|
871
|
-
case 'find_path': {
|
|
872
|
-
const result = findPath(action.target, {
|
|
873
|
-
type: action.pathType,
|
|
874
|
-
maxResults: 10
|
|
875
|
-
});
|
|
876
|
-
if (!result.success) {
|
|
877
|
-
return result.message;
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
if (action.openAfter === true) {
|
|
881
|
-
if (result.matches.length === 1) {
|
|
882
|
-
const match = result.matches[0];
|
|
883
|
-
const openResult = await openFile(match.path);
|
|
884
|
-
return openResult || `Successfully found and opened ${match.type === 'dir' ? 'folder' : 'file'}: ${match.path} ✅`;
|
|
885
|
-
}
|
|
886
|
-
return `Found multiple matches for "${action.target}". Please be more specific:\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
|
|
887
|
-
}
|
|
888
|
-
|
|
889
|
-
return `Found matches for "${action.target}":\n${result.matches.map(m => `- [${m.type}] ${m.path}`).join('\n')}`;
|
|
890
|
-
}
|
|
891
|
-
case 'clipboard_write':
|
|
892
|
-
clipboard.writeText(action.target);
|
|
893
|
-
break;
|
|
894
|
-
case 'learn_file':
|
|
895
|
-
return await indexFile(action.target);
|
|
896
|
-
case 'learn_folder':
|
|
897
|
-
const { indexFolder } = require('./src/AI_Brain/knowledge_base');
|
|
898
|
-
return await indexFolder(action.target);
|
|
899
|
-
case 'mcp_tool':
|
|
900
|
-
const mcpResult = await mcpManager.callTool(action.server, action.target, action.args);
|
|
901
|
-
return JSON.stringify(mcpResult.content);
|
|
902
|
-
case 'mouse_move':
|
|
903
|
-
return await granularAutomation.mouseMove(action.x, action.y);
|
|
904
|
-
case 'mouse_click':
|
|
905
|
-
return await granularAutomation.mouseClick(action.x, action.y, action.button || 1);
|
|
906
|
-
case 'type_text':
|
|
907
|
-
return await granularAutomation.typeText(action.target);
|
|
908
|
-
case 'key_tap':
|
|
909
|
-
return await granularAutomation.keyTap(action.target);
|
|
910
|
-
case 'plugin':
|
|
911
|
-
return await pluginManager.executePlugin(action.pluginName, action.target);
|
|
912
|
-
case 'system_automation':
|
|
913
|
-
return await handleSystemAutomation(action.target);
|
|
914
|
-
}
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
async function handleSystemAutomation(target) {
|
|
918
|
-
const [cmd, value] = target.split(':');
|
|
919
|
-
switch (cmd) {
|
|
920
|
-
case 'volume':
|
|
921
|
-
return await SystemAutomation.setVolume(parseInt(value));
|
|
922
|
-
case 'mute':
|
|
923
|
-
return await SystemAutomation.mute();
|
|
924
|
-
case 'brightness':
|
|
925
|
-
return await SystemAutomation.setBrightness(parseInt(value));
|
|
926
|
-
case 'sleep':
|
|
927
|
-
return await SystemAutomation.sleep();
|
|
928
|
-
case 'restart':
|
|
929
|
-
return await SystemAutomation.restart();
|
|
930
|
-
case 'shutdown':
|
|
931
|
-
return await SystemAutomation.shutdown();
|
|
932
|
-
case 'minimize_all':
|
|
933
|
-
return await SystemAutomation.minimizeAll();
|
|
934
|
-
default:
|
|
935
|
-
// Handle cases where target is just the command (like 'sleep')
|
|
936
|
-
if (SystemAutomation[target]) {
|
|
937
|
-
return await SystemAutomation[target]();
|
|
938
|
-
}
|
|
939
|
-
throw new Error(`Unknown system automation command: ${target}`);
|
|
940
|
-
}
|
|
941
|
-
}
|