copilot-liku-cli 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +411 -0
- package/CONFIGURATION.md +302 -0
- package/CONTRIBUTING.md +225 -0
- package/ELECTRON_README.md +121 -0
- package/INSTALLATION.md +350 -0
- package/LICENSE.md +1 -0
- package/PROJECT_STATUS.md +229 -0
- package/QUICKSTART.md +255 -0
- package/README.md +167 -0
- package/TESTING.md +274 -0
- package/package.json +61 -0
- package/scripts/start.js +30 -0
- package/src/assets/tray-icon.png +0 -0
- package/src/cli/commands/agent.js +327 -0
- package/src/cli/commands/click.js +108 -0
- package/src/cli/commands/drag.js +85 -0
- package/src/cli/commands/find.js +109 -0
- package/src/cli/commands/keys.js +132 -0
- package/src/cli/commands/mouse.js +79 -0
- package/src/cli/commands/repl.js +290 -0
- package/src/cli/commands/screenshot.js +72 -0
- package/src/cli/commands/scroll.js +74 -0
- package/src/cli/commands/start.js +67 -0
- package/src/cli/commands/type.js +57 -0
- package/src/cli/commands/wait.js +84 -0
- package/src/cli/commands/window.js +104 -0
- package/src/cli/liku.js +249 -0
- package/src/cli/util/output.js +174 -0
- package/src/main/agents/base-agent.js +410 -0
- package/src/main/agents/builder.js +484 -0
- package/src/main/agents/index.js +62 -0
- package/src/main/agents/orchestrator.js +362 -0
- package/src/main/agents/researcher.js +511 -0
- package/src/main/agents/state-manager.js +344 -0
- package/src/main/agents/supervisor.js +365 -0
- package/src/main/agents/verifier.js +452 -0
- package/src/main/ai-service.js +1633 -0
- package/src/main/index.js +2208 -0
- package/src/main/inspect-service.js +467 -0
- package/src/main/system-automation.js +1186 -0
- package/src/main/ui-automation/config.js +76 -0
- package/src/main/ui-automation/core/helpers.js +41 -0
- package/src/main/ui-automation/core/index.js +15 -0
- package/src/main/ui-automation/core/powershell.js +82 -0
- package/src/main/ui-automation/elements/finder.js +274 -0
- package/src/main/ui-automation/elements/index.js +14 -0
- package/src/main/ui-automation/elements/wait.js +66 -0
- package/src/main/ui-automation/index.js +164 -0
- package/src/main/ui-automation/interactions/element-click.js +211 -0
- package/src/main/ui-automation/interactions/high-level.js +230 -0
- package/src/main/ui-automation/interactions/index.js +47 -0
- package/src/main/ui-automation/keyboard/index.js +15 -0
- package/src/main/ui-automation/keyboard/input.js +179 -0
- package/src/main/ui-automation/mouse/click.js +186 -0
- package/src/main/ui-automation/mouse/drag.js +88 -0
- package/src/main/ui-automation/mouse/index.js +30 -0
- package/src/main/ui-automation/mouse/movement.js +51 -0
- package/src/main/ui-automation/mouse/scroll.js +116 -0
- package/src/main/ui-automation/screenshot.js +183 -0
- package/src/main/ui-automation/window/index.js +23 -0
- package/src/main/ui-automation/window/manager.js +305 -0
- package/src/main/utils/time.js +62 -0
- package/src/main/visual-awareness.js +597 -0
- package/src/renderer/chat/chat.js +671 -0
- package/src/renderer/chat/index.html +725 -0
- package/src/renderer/chat/preload.js +112 -0
- package/src/renderer/overlay/index.html +648 -0
- package/src/renderer/overlay/overlay.js +782 -0
- package/src/renderer/overlay/preload.js +90 -0
- package/src/shared/grid-math.js +82 -0
- package/src/shared/inspect-types.js +230 -0
|
@@ -0,0 +1,2208 @@
|
|
|
1
|
+
// Ensure Electron runs in app mode even if a dev shell has ELECTRON_RUN_AS_NODE set
|
|
2
|
+
if (process.env.ELECTRON_RUN_AS_NODE) {
|
|
3
|
+
console.warn('ELECTRON_RUN_AS_NODE was set; clearing so the app can start normally.');
|
|
4
|
+
delete process.env.ELECTRON_RUN_AS_NODE;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
const {
|
|
8
|
+
app,
|
|
9
|
+
BrowserWindow,
|
|
10
|
+
Tray,
|
|
11
|
+
Menu,
|
|
12
|
+
globalShortcut,
|
|
13
|
+
ipcMain,
|
|
14
|
+
screen,
|
|
15
|
+
nativeImage,
|
|
16
|
+
desktopCapturer
|
|
17
|
+
} = require('electron');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const fs = require('fs');
|
|
20
|
+
const os = require('os');
|
|
21
|
+
|
|
22
|
+
// AI Service for handling chat responses
|
|
23
|
+
const aiService = require('./ai-service.js');
|
|
24
|
+
|
|
25
|
+
// Visual awareness for advanced screen analysis
|
|
26
|
+
const visualAwareness = require('./visual-awareness.js');
|
|
27
|
+
|
|
28
|
+
// Multi-agent system for advanced AI orchestration
|
|
29
|
+
const { createAgentSystem } = require('./agents/index.js');
|
|
30
|
+
|
|
31
|
+
// Inspect service for overlay region detection and targeting
|
|
32
|
+
const inspectService = require('./inspect-service.js');
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
// Ensure caches land in a writable location to avoid Windows permission issues
|
|
36
|
+
const cacheRoot = path.join(os.tmpdir(), 'copilot-liku-electron-cache');
|
|
37
|
+
const mediaCache = path.join(cacheRoot, 'media');
|
|
38
|
+
const userDataPath = path.join(cacheRoot, 'user-data');
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
fs.mkdirSync(cacheRoot, { recursive: true });
|
|
42
|
+
fs.mkdirSync(mediaCache, { recursive: true });
|
|
43
|
+
fs.mkdirSync(userDataPath, { recursive: true });
|
|
44
|
+
|
|
45
|
+
// Force Electron to use temp-backed storage to avoid permission issues on locked-down drives
|
|
46
|
+
app.setPath('userData', userDataPath);
|
|
47
|
+
app.setPath('cache', cacheRoot);
|
|
48
|
+
|
|
49
|
+
app.commandLine.appendSwitch('disk-cache-dir', cacheRoot);
|
|
50
|
+
app.commandLine.appendSwitch('media-cache-dir', mediaCache);
|
|
51
|
+
app.commandLine.appendSwitch('disable-gpu-shader-disk-cache');
|
|
52
|
+
} catch (error) {
|
|
53
|
+
console.warn('Unable to create cache directories; continuing with defaults.', error);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Keep references to windows to prevent garbage collection
|
|
57
|
+
let overlayWindow = null;
|
|
58
|
+
let chatWindow = null;
|
|
59
|
+
let tray = null;
|
|
60
|
+
|
|
61
|
+
// State management
|
|
62
|
+
let overlayMode = 'selection'; // start in selection so the grid is visible immediately
|
|
63
|
+
let isChatVisible = false;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Create the transparent overlay window that floats above all other windows
|
|
67
|
+
*/
|
|
68
|
+
function createOverlayWindow() {
|
|
69
|
+
const { width, height } = screen.getPrimaryDisplay().bounds;
|
|
70
|
+
|
|
71
|
+
overlayWindow = new BrowserWindow({
|
|
72
|
+
width,
|
|
73
|
+
height,
|
|
74
|
+
frame: false,
|
|
75
|
+
transparent: true,
|
|
76
|
+
alwaysOnTop: true,
|
|
77
|
+
skipTaskbar: true,
|
|
78
|
+
resizable: false,
|
|
79
|
+
movable: false,
|
|
80
|
+
minimizable: false,
|
|
81
|
+
maximizable: false,
|
|
82
|
+
closable: false,
|
|
83
|
+
focusable: true,
|
|
84
|
+
hasShadow: false,
|
|
85
|
+
webPreferences: {
|
|
86
|
+
nodeIntegration: false,
|
|
87
|
+
contextIsolation: true,
|
|
88
|
+
preload: path.join(__dirname, '../renderer/overlay/preload.js')
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
// Set highest level for macOS to float above fullscreen apps
|
|
93
|
+
if (process.platform === 'darwin') {
|
|
94
|
+
overlayWindow.setAlwaysOnTop(true, 'screen-saver');
|
|
95
|
+
overlayWindow.setFullScreen(true);
|
|
96
|
+
} else {
|
|
97
|
+
// On Windows: Use maximize instead of fullscreen to avoid interfering with other windows
|
|
98
|
+
overlayWindow.setAlwaysOnTop(true, 'screen-saver');
|
|
99
|
+
overlayWindow.maximize();
|
|
100
|
+
overlayWindow.setPosition(0, 0);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Start in click-through mode
|
|
104
|
+
overlayWindow.setIgnoreMouseEvents(true, { forward: true });
|
|
105
|
+
|
|
106
|
+
overlayWindow.loadFile(path.join(__dirname, '../renderer/overlay/index.html'));
|
|
107
|
+
|
|
108
|
+
// Once the overlay loads, ensure it is visible and interactive
|
|
109
|
+
overlayWindow.webContents.on('did-finish-load', () => {
|
|
110
|
+
overlayWindow.show();
|
|
111
|
+
setOverlayMode('selection');
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// Pipe renderer console to main for debugging without DevTools
|
|
115
|
+
overlayWindow.webContents.on('console-message', (event) => {
|
|
116
|
+
const { level, message, line, sourceId } = event;
|
|
117
|
+
console.log(`[overlay console] (${level}) ${sourceId}:${line} - ${message}`);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// Prevent overlay from appearing in Dock/Taskbar
|
|
121
|
+
if (process.platform === 'darwin') {
|
|
122
|
+
app.dock.hide();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
overlayWindow.on('closed', () => {
|
|
126
|
+
overlayWindow = null;
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Chat window position preferences (persisted)
|
|
131
|
+
let chatBoundsPrefs = null;
|
|
132
|
+
|
|
133
|
+
function loadChatBoundsPrefs() {
|
|
134
|
+
try {
|
|
135
|
+
const prefsPath = path.join(userDataPath, 'chat-bounds.json');
|
|
136
|
+
if (fs.existsSync(prefsPath)) {
|
|
137
|
+
chatBoundsPrefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8'));
|
|
138
|
+
console.log('Loaded chat bounds preferences:', chatBoundsPrefs);
|
|
139
|
+
}
|
|
140
|
+
} catch (e) {
|
|
141
|
+
console.warn('Could not load chat bounds preferences:', e);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function saveChatBoundsPrefs(bounds) {
|
|
146
|
+
try {
|
|
147
|
+
const prefsPath = path.join(userDataPath, 'chat-bounds.json');
|
|
148
|
+
fs.writeFileSync(prefsPath, JSON.stringify(bounds));
|
|
149
|
+
chatBoundsPrefs = bounds;
|
|
150
|
+
} catch (e) {
|
|
151
|
+
console.warn('Could not save chat bounds preferences:', e);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Create the chat window positioned at screen edge (bottom-right)
|
|
157
|
+
* FRESH APPROACH: Create window with absolute minimal config, position AFTER creation
|
|
158
|
+
*/
|
|
159
|
+
function createChatWindow() {
|
|
160
|
+
// Destroy existing window if any
|
|
161
|
+
if (chatWindow) {
|
|
162
|
+
chatWindow.destroy();
|
|
163
|
+
chatWindow = null;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const display = screen.getPrimaryDisplay();
|
|
167
|
+
const { width: screenWidth, height: screenHeight } = display.workAreaSize;
|
|
168
|
+
|
|
169
|
+
// HARDCODED small window - bottom right
|
|
170
|
+
const W = 380;
|
|
171
|
+
const H = 500;
|
|
172
|
+
const X = screenWidth - W - 20;
|
|
173
|
+
const Y = screenHeight - H - 20;
|
|
174
|
+
|
|
175
|
+
console.log(`[CHAT] Creating at ${X},${Y} size ${W}x${H}`);
|
|
176
|
+
|
|
177
|
+
chatWindow = new BrowserWindow({
|
|
178
|
+
width: W,
|
|
179
|
+
height: H,
|
|
180
|
+
x: X,
|
|
181
|
+
y: Y,
|
|
182
|
+
minWidth: 300,
|
|
183
|
+
minHeight: 400,
|
|
184
|
+
maxWidth: 600,
|
|
185
|
+
maxHeight: 800,
|
|
186
|
+
frame: false,
|
|
187
|
+
transparent: false,
|
|
188
|
+
resizable: true,
|
|
189
|
+
minimizable: true,
|
|
190
|
+
maximizable: false,
|
|
191
|
+
fullscreenable: false,
|
|
192
|
+
alwaysOnTop: false,
|
|
193
|
+
skipTaskbar: false,
|
|
194
|
+
show: false,
|
|
195
|
+
backgroundColor: '#1e1e1e',
|
|
196
|
+
webPreferences: {
|
|
197
|
+
nodeIntegration: false,
|
|
198
|
+
contextIsolation: true,
|
|
199
|
+
preload: path.join(__dirname, '../renderer/chat/preload.js')
|
|
200
|
+
}
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
// Immediately set bounds again
|
|
204
|
+
chatWindow.setBounds({ x: X, y: Y, width: W, height: H });
|
|
205
|
+
|
|
206
|
+
chatWindow.loadFile(path.join(__dirname, '../renderer/chat/index.html'));
|
|
207
|
+
|
|
208
|
+
const persistBounds = () => {
|
|
209
|
+
if (!chatWindow) return;
|
|
210
|
+
saveChatBoundsPrefs(chatWindow.getBounds());
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
chatWindow.webContents.on('did-finish-load', () => {
|
|
214
|
+
// Force bounds one more time after load
|
|
215
|
+
chatWindow.setBounds({ x: X, y: Y, width: W, height: H });
|
|
216
|
+
console.log(`[CHAT] Loaded. Bounds: ${JSON.stringify(chatWindow.getBounds())}`);
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
chatWindow.on('resize', persistBounds);
|
|
220
|
+
chatWindow.on('move', persistBounds);
|
|
221
|
+
|
|
222
|
+
chatWindow.on('close', (event) => {
|
|
223
|
+
if (!app.isQuitting) {
|
|
224
|
+
event.preventDefault();
|
|
225
|
+
chatWindow.hide();
|
|
226
|
+
isChatVisible = false;
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
chatWindow.on('closed', () => {
|
|
231
|
+
chatWindow = null;
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Toggle chat - recreate window fresh each time to avoid fullscreen bug
|
|
237
|
+
*/
|
|
238
|
+
function toggleChat() {
|
|
239
|
+
if (chatWindow && chatWindow.isVisible()) {
|
|
240
|
+
chatWindow.hide();
|
|
241
|
+
isChatVisible = false;
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// RECREATE window fresh each time
|
|
246
|
+
createChatWindow();
|
|
247
|
+
|
|
248
|
+
// Show after a brief delay to ensure bounds are set
|
|
249
|
+
setTimeout(() => {
|
|
250
|
+
if (chatWindow) {
|
|
251
|
+
const display = screen.getPrimaryDisplay();
|
|
252
|
+
const { width: screenWidth, height: screenHeight } = display.workAreaSize;
|
|
253
|
+
const W = 380, H = 500;
|
|
254
|
+
const X = screenWidth - W - 20;
|
|
255
|
+
const Y = screenHeight - H - 20;
|
|
256
|
+
|
|
257
|
+
// AGGRESSIVE: Multiple setters to override any system defaults
|
|
258
|
+
chatWindow.unmaximize();
|
|
259
|
+
chatWindow.setFullScreen(false);
|
|
260
|
+
chatWindow.setSize(W, H);
|
|
261
|
+
chatWindow.setPosition(X, Y);
|
|
262
|
+
chatWindow.setBounds({ x: X, y: Y, width: W, height: H });
|
|
263
|
+
|
|
264
|
+
chatWindow.show();
|
|
265
|
+
chatWindow.focus();
|
|
266
|
+
|
|
267
|
+
// AFTER show: force bounds again
|
|
268
|
+
chatWindow.setSize(W, H);
|
|
269
|
+
chatWindow.setPosition(X, Y);
|
|
270
|
+
|
|
271
|
+
isChatVisible = true;
|
|
272
|
+
console.log(`[CHAT] Shown. Final bounds: ${JSON.stringify(chatWindow.getBounds())}`);
|
|
273
|
+
|
|
274
|
+
// Validate bounds after 200ms and correct if needed
|
|
275
|
+
setTimeout(() => {
|
|
276
|
+
if (chatWindow) {
|
|
277
|
+
const bounds = chatWindow.getBounds();
|
|
278
|
+
if (bounds.width !== W || bounds.height !== H) {
|
|
279
|
+
console.log(`[CHAT] CORRECTING: Bounds were ${JSON.stringify(bounds)}, forcing to ${W}x${H}@${X},${Y}`);
|
|
280
|
+
chatWindow.setSize(W, H);
|
|
281
|
+
chatWindow.setPosition(X, Y);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}, 200);
|
|
285
|
+
}
|
|
286
|
+
}, 100);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Create system tray icon with menu
|
|
291
|
+
*/
|
|
292
|
+
function loadTrayIcon() {
|
|
293
|
+
const candidates = [
|
|
294
|
+
path.join(__dirname, '../assets/tray-icon.png'),
|
|
295
|
+
path.join(app.getAppPath(), 'src/assets/tray-icon.png'),
|
|
296
|
+
path.join(process.resourcesPath, 'assets', 'tray-icon.png'),
|
|
297
|
+
path.join(process.resourcesPath, 'tray-icon.png')
|
|
298
|
+
];
|
|
299
|
+
|
|
300
|
+
for (const candidate of candidates) {
|
|
301
|
+
try {
|
|
302
|
+
if (!fs.existsSync(candidate)) {
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const image = nativeImage.createFromPath(candidate);
|
|
307
|
+
|
|
308
|
+
if (!image.isEmpty()) {
|
|
309
|
+
return { image, source: candidate };
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
console.warn(`Tray icon candidate was empty: ${candidate}`);
|
|
313
|
+
} catch (error) {
|
|
314
|
+
console.warn(`Tray icon candidate failed (${candidate}):`, error);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const fallbackBase64 = 'iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAASbSURBVFhHxZf7U1R1FMDv34E8hGW5LBcMAiQiQkMgogVqYW1FA1EDRF6Rg5oMp0hn0CYQB4KBFAlGGoQZgtmaxmqRQWCkSIpVA7aUjZeJkI8RrTnN+a57Y5e7j/xlzy935nvvPZ/z/n6/nCDEbAkK2gpBz8WCEJIAQkgiCKFKEEJTQAhXgRCeDkKEBhSRGaCIzARF1G5QRO0DRXQO8NH5wMcUAB9TAvzWUuBjy4CPPQx8XDnwcRUgT6gEecIxkCdWgTzpJMiTqkGmPAUyZR3IUhpBltIEHIMHb8PAkAQMfP41DAxVYmBYKgrhKhQ2q1GI0GBA5E4MeDETFVHZqHhpHyqic9H/5Xz0jylE/y0l6P/Ke8jHliG/7QjyceXIxwP6JVSi36vH0S+xCuVJH6P89Wr0Vdaib3Id+qY0oiy1GWVvnEWOPHcVXPZmK3IUdlfBZWntZEAiuAruk96BHBWcq+A+6k4ygFW7S+A+27uRY632jPDUvAbcebAV1cXnngnu81YPGcD63Gn43sMtOPKTAaVk7PosftR0yWm4t6YPORoyzsA3Jb6PLV2DFkDj/DKOXL2J0zN3LNb7f7iFL2S1O4R77/gKOZpwzsD1U7MioLFjEMNUn1iEXVDVYXm9Dlfur7Jv6Blf0GMX7p3xDRlA49U2nMJ+vneEKTXO30V10Rm7BUeeD/88x74fu3EbFTs6bcI37voWOTbb7cDzK9qYskerT9bBkwvbMONIFyqLvrDIecD2dpxZvM/+O9U5YRO+8W0dGUAbizScql0/ZfLm9Of9IjxUfRr7R397mhCTjN1YwPgD3WLONfCdyfDHf6NiV68k3CtzADna1WzBNyV9IAKCkk+Ing+Pz6xB/ydTxmUM0HSIOZ/64y+2nnTokiTcK+sycrSlSsGpz9VFzUyBfnpehKeXdlhhLSXnxICY895BI1s7+tkvknCv3SPI0X4uBacJV1GrZQr6dHqx4MrrTaG1JWtzXnX+GltruzgjCffMHiUDSkAKTuN1z1GTt2PXZsVqz4ZeK6SlHKwfFQuu7eIttvZh26+ScM89Y8g9Pcmsg1PBhaXVMAXUAeZWE9KbcWbhnhXWJFRwEblfiwU38bupBjJPjkvCPfeOI8eOURJwc86NCytMSd5xrdhqqrI+XLz7cB28sHZUhG/er8NHj/9h74LyhiThnu9MkAHsDCcJp7A3dI4yJYtLDzAko1UcryGZnVjZ8iPLeeW5cQvPqeB0V/9k/+nGl2zCPXKuI8cOkDbgFHYhrQH1BpOy3gGDw9lOcMo5ycqDJxhWdMUm3CN3kgyoAFtw83iNz7/AQkxC4zWuWCsJp7CbPScpbZ60C/fIMyBHR2d7cPN4VR3SiuOVZHjiNp7RTrNW+7RnGof0S2LOyXNn4B77b5IBx8AR3Bz24KwuvPC95Qi2Fsq5o7Cb4e75RuRMlwbH8LU5D87WoqZyCGu6JvHLy3NY021grRZVesVuwVnD3Q/MIcduLP8Dbl1wtiacM3D3gkUyoBpcBd9QdAc5uqu5Cr6heJkMqANXwd1K7iHHbqkugru9+5AMaAJXwd1KV/Ff/Hw4CMaLXiMAAAAASUVORK5CYII=';
|
|
319
|
+
const fallbackImage = nativeImage.createFromDataURL(`data:image/png;base64,${fallbackBase64}`);
|
|
320
|
+
|
|
321
|
+
return { image: fallbackImage, source: 'embedded-fallback' };
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function createTray() {
|
|
325
|
+
const { image: trayIcon, source } = loadTrayIcon();
|
|
326
|
+
|
|
327
|
+
try {
|
|
328
|
+
tray = new Tray(trayIcon);
|
|
329
|
+
} catch (error) {
|
|
330
|
+
console.error('Failed to initialize tray icon:', error);
|
|
331
|
+
tray = new Tray(nativeImage.createEmpty());
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
if (source === 'embedded-fallback') {
|
|
335
|
+
console.warn('Using embedded fallback tray icon because no valid asset was found.');
|
|
336
|
+
} else {
|
|
337
|
+
console.log(`Tray icon loaded from: ${source}`);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const contextMenu = Menu.buildFromTemplate([
|
|
341
|
+
{
|
|
342
|
+
label: 'Open Chat',
|
|
343
|
+
click: () => toggleChat()
|
|
344
|
+
},
|
|
345
|
+
{
|
|
346
|
+
label: 'Toggle Overlay',
|
|
347
|
+
click: () => toggleOverlay()
|
|
348
|
+
},
|
|
349
|
+
{ type: 'separator' },
|
|
350
|
+
{
|
|
351
|
+
label: 'Reset Window Positions',
|
|
352
|
+
click: () => {
|
|
353
|
+
// Clear saved preferences and reset both windows
|
|
354
|
+
chatBoundsPrefs = null;
|
|
355
|
+
try {
|
|
356
|
+
const prefsPath = path.join(userDataPath, 'chat-bounds.json');
|
|
357
|
+
if (fs.existsSync(prefsPath)) fs.unlinkSync(prefsPath);
|
|
358
|
+
} catch (e) {}
|
|
359
|
+
ensureChatBounds(true);
|
|
360
|
+
if (chatWindow && chatWindow.isVisible()) {
|
|
361
|
+
chatWindow.show();
|
|
362
|
+
chatWindow.focus();
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
},
|
|
366
|
+
{ type: 'separator' },
|
|
367
|
+
{
|
|
368
|
+
label: 'Quit',
|
|
369
|
+
click: () => {
|
|
370
|
+
app.isQuitting = true;
|
|
371
|
+
app.quit();
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
]);
|
|
375
|
+
|
|
376
|
+
tray.setToolTip('Copilot Agent Overlay');
|
|
377
|
+
tray.setContextMenu(contextMenu);
|
|
378
|
+
|
|
379
|
+
// On macOS, clicking tray icon shows chat
|
|
380
|
+
tray.on('click', () => {
|
|
381
|
+
toggleChat();
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Ensure chat window has valid bounds (not off-screen, not fullscreen)
|
|
387
|
+
*/
|
|
388
|
+
function ensureChatBounds(force = false) {
|
|
389
|
+
if (!chatWindow) return;
|
|
390
|
+
|
|
391
|
+
// Always ensure not fullscreen
|
|
392
|
+
if (chatWindow.isFullScreen()) {
|
|
393
|
+
chatWindow.setFullScreen(false);
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
const { width, height } = screen.getPrimaryDisplay().workAreaSize;
|
|
397
|
+
const bounds = chatWindow.getBounds();
|
|
398
|
+
|
|
399
|
+
// Check if off-screen
|
|
400
|
+
const isOffScreen = bounds.x < -bounds.width ||
|
|
401
|
+
bounds.x > width ||
|
|
402
|
+
bounds.y < -bounds.height ||
|
|
403
|
+
bounds.y > height;
|
|
404
|
+
|
|
405
|
+
// Check if too large for screen
|
|
406
|
+
const isTooLarge = bounds.width > width || bounds.height > height;
|
|
407
|
+
|
|
408
|
+
if (force || isOffScreen || isTooLarge) {
|
|
409
|
+
if (chatWindow.isMaximized()) {
|
|
410
|
+
chatWindow.unmaximize();
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
// Use saved preferences or calculate default bottom-right position
|
|
414
|
+
const defaultWidth = chatBoundsPrefs?.width || 380;
|
|
415
|
+
const defaultHeight = chatBoundsPrefs?.height || 520;
|
|
416
|
+
const margin = 20;
|
|
417
|
+
|
|
418
|
+
chatWindow.setBounds({
|
|
419
|
+
width: Math.min(defaultWidth, width - margin * 2),
|
|
420
|
+
height: Math.min(defaultHeight, height - margin * 2),
|
|
421
|
+
x: chatBoundsPrefs?.x ?? Math.max(0, width - defaultWidth - margin),
|
|
422
|
+
y: chatBoundsPrefs?.y ?? Math.max(0, height - defaultHeight - margin)
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* Toggle overlay visibility
|
|
429
|
+
*/
|
|
430
|
+
function toggleOverlay() {
|
|
431
|
+
if (!overlayWindow) return;
|
|
432
|
+
|
|
433
|
+
if (overlayWindow.isVisible()) {
|
|
434
|
+
overlayWindow.hide();
|
|
435
|
+
setOverlayMode('passive');
|
|
436
|
+
} else {
|
|
437
|
+
overlayWindow.show();
|
|
438
|
+
setOverlayMode('selection');
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* Set overlay mode (passive or selection)
|
|
444
|
+
*
|
|
445
|
+
* CRITICAL: We ALWAYS use setIgnoreMouseEvents(true, { forward: true }) so that
|
|
446
|
+
* clicks pass through to background applications. The overlay dots use CSS
|
|
447
|
+
* pointer-events: auto to still receive clicks when hovered. This is the
|
|
448
|
+
* correct pattern for transparent overlays with clickable elements.
|
|
449
|
+
*/
|
|
450
|
+
function setOverlayMode(mode) {
|
|
451
|
+
overlayMode = mode;
|
|
452
|
+
|
|
453
|
+
if (!overlayWindow) return;
|
|
454
|
+
|
|
455
|
+
// ALWAYS forward mouse events to apps beneath the overlay.
|
|
456
|
+
// Dots with pointer-events: auto in CSS will still receive clicks.
|
|
457
|
+
overlayWindow.setIgnoreMouseEvents(true, { forward: true });
|
|
458
|
+
|
|
459
|
+
if (mode === 'passive') {
|
|
460
|
+
overlayWindow.setFocusable(false);
|
|
461
|
+
unregisterOverlayShortcuts();
|
|
462
|
+
} else if (mode === 'selection') {
|
|
463
|
+
// In selection mode, allow the window to be focusable for keyboard events
|
|
464
|
+
if (typeof overlayWindow.setFocusable === 'function') {
|
|
465
|
+
overlayWindow.setFocusable(true);
|
|
466
|
+
}
|
|
467
|
+
registerOverlayShortcuts();
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// Notify overlay renderer of mode change
|
|
471
|
+
overlayWindow.webContents.send('mode-changed', mode);
|
|
472
|
+
console.log(`Overlay mode set to ${mode} (click-through enabled, dots are clickable via CSS)`);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Register overlay-specific shortcuts when in selection mode
|
|
477
|
+
* These use globalShortcut because the overlay has setIgnoreMouseEvents(true)
|
|
478
|
+
* which means keyboard events go to background apps, not the overlay window
|
|
479
|
+
*/
|
|
480
|
+
function registerOverlayShortcuts() {
|
|
481
|
+
console.log('[SHORTCUTS] Registering overlay shortcuts (Ctrl+Alt+F/G/+/-/X/I)');
|
|
482
|
+
|
|
483
|
+
// Ctrl+Alt+F to toggle fine grid
|
|
484
|
+
globalShortcut.register('CommandOrControl+Alt+F', () => {
|
|
485
|
+
if (overlayWindow && overlayMode === 'selection') {
|
|
486
|
+
console.log('[SHORTCUTS] Ctrl+Alt+F pressed - toggle fine grid');
|
|
487
|
+
console.log('[SHORTCUTS] overlayWindow destroyed?', overlayWindow.isDestroyed());
|
|
488
|
+
console.log('[SHORTCUTS] Sending overlay-command to webContents');
|
|
489
|
+
overlayWindow.webContents.send('overlay-command', { action: 'toggle-fine' });
|
|
490
|
+
console.log('[SHORTCUTS] Sent overlay-command');
|
|
491
|
+
} else {
|
|
492
|
+
console.log('[SHORTCUTS] Ctrl+Alt+F pressed but not in selection mode or no overlay');
|
|
493
|
+
}
|
|
494
|
+
});
|
|
495
|
+
|
|
496
|
+
// Ctrl+Alt+G to show all grids
|
|
497
|
+
globalShortcut.register('CommandOrControl+Alt+G', () => {
|
|
498
|
+
if (overlayWindow && overlayMode === 'selection') {
|
|
499
|
+
console.log('[SHORTCUTS] Ctrl+Alt+G pressed - show all grids');
|
|
500
|
+
overlayWindow.webContents.send('overlay-command', { action: 'show-all' });
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
// Ctrl+Alt+= to zoom in
|
|
505
|
+
globalShortcut.register('CommandOrControl+Alt+=', () => {
|
|
506
|
+
if (overlayWindow && overlayMode === 'selection') {
|
|
507
|
+
console.log('[SHORTCUTS] Ctrl+Alt+= pressed - zoom in');
|
|
508
|
+
overlayWindow.webContents.send('overlay-command', { action: 'zoom-in' });
|
|
509
|
+
}
|
|
510
|
+
});
|
|
511
|
+
|
|
512
|
+
// Ctrl+Alt+- to zoom out
|
|
513
|
+
globalShortcut.register('CommandOrControl+Alt+-', () => {
|
|
514
|
+
if (overlayWindow && overlayMode === 'selection') {
|
|
515
|
+
console.log('[SHORTCUTS] Ctrl+Alt+- pressed - zoom out');
|
|
516
|
+
overlayWindow.webContents.send('overlay-command', { action: 'zoom-out' });
|
|
517
|
+
}
|
|
518
|
+
});
|
|
519
|
+
|
|
520
|
+
// Ctrl+Alt+X to cancel selection
|
|
521
|
+
globalShortcut.register('CommandOrControl+Alt+X', () => {
|
|
522
|
+
if (overlayWindow && overlayMode === 'selection') {
|
|
523
|
+
console.log('[SHORTCUTS] Ctrl+Alt+X pressed - cancel');
|
|
524
|
+
overlayWindow.webContents.send('overlay-command', { action: 'cancel' });
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
|
|
528
|
+
// Ctrl+Alt+I to toggle inspect mode
|
|
529
|
+
globalShortcut.register('CommandOrControl+Alt+I', () => {
|
|
530
|
+
if (overlayWindow && overlayMode === 'selection') {
|
|
531
|
+
console.log('[SHORTCUTS] Ctrl+Alt+I pressed - toggle inspect mode');
|
|
532
|
+
// Toggle inspect mode via IPC
|
|
533
|
+
const newState = !inspectService.isInspectModeActive();
|
|
534
|
+
inspectService.setInspectMode(newState);
|
|
535
|
+
|
|
536
|
+
// Notify overlay
|
|
537
|
+
overlayWindow.webContents.send('inspect-mode-changed', newState);
|
|
538
|
+
overlayWindow.webContents.send('overlay-command', { action: 'toggle-inspect' });
|
|
539
|
+
|
|
540
|
+
// If enabled, trigger region detection
|
|
541
|
+
if (newState) {
|
|
542
|
+
// Use async detection with error handling
|
|
543
|
+
inspectService.detectRegions().then(results => {
|
|
544
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
545
|
+
overlayWindow.webContents.send('inspect-regions-update', results.regions);
|
|
546
|
+
}
|
|
547
|
+
}).catch(err => {
|
|
548
|
+
console.error('[SHORTCUTS] Inspect region detection failed:', err);
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/**
|
|
556
|
+
* Unregister overlay-specific shortcuts when leaving selection mode
|
|
557
|
+
*/
|
|
558
|
+
function unregisterOverlayShortcuts() {
|
|
559
|
+
console.log('[SHORTCUTS] Unregistering overlay shortcuts');
|
|
560
|
+
const keys = [
|
|
561
|
+
'CommandOrControl+Alt+F',
|
|
562
|
+
'CommandOrControl+Alt+G',
|
|
563
|
+
'CommandOrControl+Alt+=',
|
|
564
|
+
'CommandOrControl+Alt+-',
|
|
565
|
+
'CommandOrControl+Alt+X',
|
|
566
|
+
'CommandOrControl+Alt+I'
|
|
567
|
+
];
|
|
568
|
+
keys.forEach(key => {
|
|
569
|
+
try {
|
|
570
|
+
globalShortcut.unregister(key);
|
|
571
|
+
} catch (e) {
|
|
572
|
+
// Ignore errors if shortcut wasn't registered
|
|
573
|
+
}
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
/**
|
|
578
|
+
* Register global shortcuts
|
|
579
|
+
*/
|
|
580
|
+
function registerShortcuts() {
|
|
581
|
+
// Ctrl+Alt+Space to toggle chat
|
|
582
|
+
globalShortcut.register('CommandOrControl+Alt+Space', () => {
|
|
583
|
+
toggleChat();
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
// Ctrl+Shift+O to toggle overlay
|
|
587
|
+
globalShortcut.register('CommandOrControl+Shift+O', () => {
|
|
588
|
+
toggleOverlay();
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* Set up IPC handlers
|
|
594
|
+
*/
|
|
595
|
+
function setupIPC() {
|
|
596
|
+
// Handle dot selection from overlay
|
|
597
|
+
ipcMain.on('dot-selected', (event, data) => {
|
|
598
|
+
console.log('Dot selected:', data);
|
|
599
|
+
|
|
600
|
+
// Forward to chat window
|
|
601
|
+
if (chatWindow) {
|
|
602
|
+
chatWindow.webContents.send('dot-selected', data);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Switch back to passive mode after selection (unless cancelled)
|
|
606
|
+
if (!data.cancelled) {
|
|
607
|
+
setOverlayMode('passive');
|
|
608
|
+
}
|
|
609
|
+
});
|
|
610
|
+
|
|
611
|
+
// Handle mode change requests from chat
|
|
612
|
+
ipcMain.on('set-mode', (event, mode) => {
|
|
613
|
+
setOverlayMode(mode);
|
|
614
|
+
});
|
|
615
|
+
|
|
616
|
+
// Agentic mode flag (when true, actions execute automatically)
|
|
617
|
+
let agenticMode = false;
|
|
618
|
+
let pendingActions = null;
|
|
619
|
+
|
|
620
|
+
// Handle chat messages
|
|
621
|
+
ipcMain.on('chat-message', async (event, message) => {
|
|
622
|
+
console.log('Chat message:', message);
|
|
623
|
+
|
|
624
|
+
// Check for slash commands first
|
|
625
|
+
if (message.startsWith('/')) {
|
|
626
|
+
// Handle agentic mode toggle
|
|
627
|
+
if (message === '/agentic' || message === '/agent') {
|
|
628
|
+
agenticMode = !agenticMode;
|
|
629
|
+
if (chatWindow) {
|
|
630
|
+
chatWindow.webContents.send('agent-response', {
|
|
631
|
+
text: `Agentic mode ${agenticMode ? 'ENABLED' : 'DISABLED'}. ${agenticMode ? 'Actions will execute automatically.' : 'Actions will require confirmation.'}`,
|
|
632
|
+
type: 'system',
|
|
633
|
+
timestamp: Date.now()
|
|
634
|
+
});
|
|
635
|
+
}
|
|
636
|
+
return;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
// ===== MULTI-AGENT SYSTEM COMMANDS =====
|
|
640
|
+
// /orchestrate - Run full orchestration on a task
|
|
641
|
+
if (message.startsWith('/orchestrate ')) {
|
|
642
|
+
const task = message.slice('/orchestrate '.length).trim();
|
|
643
|
+
if (chatWindow) {
|
|
644
|
+
chatWindow.webContents.send('agent-response', {
|
|
645
|
+
text: `🎭 Starting multi-agent orchestration for: "${task}"`,
|
|
646
|
+
type: 'system',
|
|
647
|
+
timestamp: Date.now()
|
|
648
|
+
});
|
|
649
|
+
chatWindow.webContents.send('agent-typing', { isTyping: true });
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
try {
|
|
653
|
+
const { orchestrator } = getAgentSystem();
|
|
654
|
+
const result = await orchestrator.orchestrate(task);
|
|
655
|
+
|
|
656
|
+
if (chatWindow) {
|
|
657
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
658
|
+
chatWindow.webContents.send('agent-response', {
|
|
659
|
+
text: `🎭 Orchestration complete:\n\n${JSON.stringify(result, null, 2)}`,
|
|
660
|
+
type: result.status === 'success' ? 'message' : 'error',
|
|
661
|
+
timestamp: Date.now()
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
} catch (error) {
|
|
665
|
+
if (chatWindow) {
|
|
666
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
667
|
+
chatWindow.webContents.send('agent-response', {
|
|
668
|
+
text: `❌ Orchestration failed: ${error.message}`,
|
|
669
|
+
type: 'error',
|
|
670
|
+
timestamp: Date.now()
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
return;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
// /research - Use researcher agent
|
|
678
|
+
if (message.startsWith('/research ')) {
|
|
679
|
+
const query = message.slice('/research '.length).trim();
|
|
680
|
+
if (chatWindow) {
|
|
681
|
+
chatWindow.webContents.send('agent-response', {
|
|
682
|
+
text: `🔍 Researching: "${query}"`,
|
|
683
|
+
type: 'system',
|
|
684
|
+
timestamp: Date.now()
|
|
685
|
+
});
|
|
686
|
+
chatWindow.webContents.send('agent-typing', { isTyping: true });
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
try {
|
|
690
|
+
const { orchestrator } = getAgentSystem();
|
|
691
|
+
const result = await orchestrator.research(query);
|
|
692
|
+
|
|
693
|
+
if (chatWindow) {
|
|
694
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
695
|
+
chatWindow.webContents.send('agent-response', {
|
|
696
|
+
text: result.findings?.length > 0
|
|
697
|
+
? `🔍 Research findings:\n\n${result.findings.join('\n\n')}`
|
|
698
|
+
: `🔍 No findings for query.`,
|
|
699
|
+
type: 'message',
|
|
700
|
+
timestamp: Date.now()
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
} catch (error) {
|
|
704
|
+
if (chatWindow) {
|
|
705
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
706
|
+
chatWindow.webContents.send('agent-response', {
|
|
707
|
+
text: `❌ Research failed: ${error.message}`,
|
|
708
|
+
type: 'error',
|
|
709
|
+
timestamp: Date.now()
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
// /build - Use builder agent
|
|
717
|
+
if (message.startsWith('/build ')) {
|
|
718
|
+
const spec = message.slice('/build '.length).trim();
|
|
719
|
+
if (chatWindow) {
|
|
720
|
+
chatWindow.webContents.send('agent-response', {
|
|
721
|
+
text: `🔨 Starting build: "${spec}"`,
|
|
722
|
+
type: 'system',
|
|
723
|
+
timestamp: Date.now()
|
|
724
|
+
});
|
|
725
|
+
chatWindow.webContents.send('agent-typing', { isTyping: true });
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
try {
|
|
729
|
+
const { orchestrator } = getAgentSystem();
|
|
730
|
+
const result = await orchestrator.build(spec);
|
|
731
|
+
|
|
732
|
+
if (chatWindow) {
|
|
733
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
734
|
+
chatWindow.webContents.send('agent-response', {
|
|
735
|
+
text: `🔨 Build complete:\n\n${JSON.stringify(result, null, 2)}`,
|
|
736
|
+
type: result.status === 'success' ? 'message' : 'error',
|
|
737
|
+
timestamp: Date.now()
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
} catch (error) {
|
|
741
|
+
if (chatWindow) {
|
|
742
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
743
|
+
chatWindow.webContents.send('agent-response', {
|
|
744
|
+
text: `❌ Build failed: ${error.message}`,
|
|
745
|
+
type: 'error',
|
|
746
|
+
timestamp: Date.now()
|
|
747
|
+
});
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
// /verify - Use verifier agent
|
|
754
|
+
if (message.startsWith('/verify ')) {
|
|
755
|
+
const target = message.slice('/verify '.length).trim();
|
|
756
|
+
if (chatWindow) {
|
|
757
|
+
chatWindow.webContents.send('agent-response', {
|
|
758
|
+
text: `✅ Verifying: "${target}"`,
|
|
759
|
+
type: 'system',
|
|
760
|
+
timestamp: Date.now()
|
|
761
|
+
});
|
|
762
|
+
chatWindow.webContents.send('agent-typing', { isTyping: true });
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
try {
|
|
766
|
+
const { orchestrator } = getAgentSystem();
|
|
767
|
+
const result = await orchestrator.verify(target);
|
|
768
|
+
|
|
769
|
+
if (chatWindow) {
|
|
770
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
771
|
+
chatWindow.webContents.send('agent-response', {
|
|
772
|
+
text: `✅ Verification results:\n\n${JSON.stringify(result, null, 2)}`,
|
|
773
|
+
type: result.passed ? 'message' : 'error',
|
|
774
|
+
timestamp: Date.now()
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
} catch (error) {
|
|
778
|
+
if (chatWindow) {
|
|
779
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
780
|
+
chatWindow.webContents.send('agent-response', {
|
|
781
|
+
text: `❌ Verification failed: ${error.message}`,
|
|
782
|
+
type: 'error',
|
|
783
|
+
timestamp: Date.now()
|
|
784
|
+
});
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
// /agent-status - Get multi-agent system status
|
|
791
|
+
if (message === '/agent-status' || message === '/agents') {
|
|
792
|
+
try {
|
|
793
|
+
const { stateManager, orchestrator } = getAgentSystem();
|
|
794
|
+
const state = stateManager.getState();
|
|
795
|
+
const currentSession = orchestrator.currentSession;
|
|
796
|
+
|
|
797
|
+
const statusText = `
|
|
798
|
+
🤖 **Multi-Agent System Status**
|
|
799
|
+
|
|
800
|
+
**Session:** ${currentSession || 'No active session'}
|
|
801
|
+
**Task Queue:** ${state.taskQueue.length} pending
|
|
802
|
+
**Completed:** ${state.completedTasks.length}
|
|
803
|
+
**Failed:** ${state.failedTasks.length}
|
|
804
|
+
**Handoffs:** ${state.handoffs.length}
|
|
805
|
+
|
|
806
|
+
**Available Commands:**
|
|
807
|
+
• \`/orchestrate <task>\` - Full multi-agent task execution
|
|
808
|
+
• \`/research <query>\` - Research using RLC patterns
|
|
809
|
+
• \`/build <spec>\` - Build code with builder agent
|
|
810
|
+
• \`/verify <target>\` - Verify code/changes
|
|
811
|
+
• \`/agent-reset\` - Reset agent system state
|
|
812
|
+
`;
|
|
813
|
+
|
|
814
|
+
if (chatWindow) {
|
|
815
|
+
chatWindow.webContents.send('agent-response', {
|
|
816
|
+
text: statusText,
|
|
817
|
+
type: 'system',
|
|
818
|
+
timestamp: Date.now()
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
} catch (error) {
|
|
822
|
+
if (chatWindow) {
|
|
823
|
+
chatWindow.webContents.send('agent-response', {
|
|
824
|
+
text: `❌ Failed to get status: ${error.message}`,
|
|
825
|
+
type: 'error',
|
|
826
|
+
timestamp: Date.now()
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
// /agent-reset - Reset multi-agent system
|
|
834
|
+
if (message === '/agent-reset') {
|
|
835
|
+
try {
|
|
836
|
+
const { stateManager } = getAgentSystem();
|
|
837
|
+
stateManager.resetState();
|
|
838
|
+
agentSystem = null;
|
|
839
|
+
|
|
840
|
+
if (chatWindow) {
|
|
841
|
+
chatWindow.webContents.send('agent-response', {
|
|
842
|
+
text: '🔄 Multi-agent system reset successfully.',
|
|
843
|
+
type: 'system',
|
|
844
|
+
timestamp: Date.now()
|
|
845
|
+
});
|
|
846
|
+
}
|
|
847
|
+
} catch (error) {
|
|
848
|
+
if (chatWindow) {
|
|
849
|
+
chatWindow.webContents.send('agent-response', {
|
|
850
|
+
text: `❌ Reset failed: ${error.message}`,
|
|
851
|
+
type: 'error',
|
|
852
|
+
timestamp: Date.now()
|
|
853
|
+
});
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
let commandResult = aiService.handleCommand(message);
|
|
860
|
+
|
|
861
|
+
// Handle async commands (like /login)
|
|
862
|
+
if (commandResult && typeof commandResult.then === 'function') {
|
|
863
|
+
commandResult = await commandResult;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
if (commandResult) {
|
|
867
|
+
if (chatWindow) {
|
|
868
|
+
chatWindow.webContents.send('agent-response', {
|
|
869
|
+
text: commandResult.message,
|
|
870
|
+
type: commandResult.type,
|
|
871
|
+
timestamp: Date.now()
|
|
872
|
+
});
|
|
873
|
+
}
|
|
874
|
+
return;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// Check if we should include visual context (expanded triggers for agentic actions)
|
|
879
|
+
const includeVisualContext = message.toLowerCase().includes('screen') ||
|
|
880
|
+
message.toLowerCase().includes('see') ||
|
|
881
|
+
message.toLowerCase().includes('look') ||
|
|
882
|
+
message.toLowerCase().includes('show') ||
|
|
883
|
+
message.toLowerCase().includes('capture') ||
|
|
884
|
+
message.toLowerCase().includes('click') ||
|
|
885
|
+
message.toLowerCase().includes('type') ||
|
|
886
|
+
message.toLowerCase().includes('print') ||
|
|
887
|
+
message.toLowerCase().includes('open') ||
|
|
888
|
+
message.toLowerCase().includes('close') ||
|
|
889
|
+
visualContextHistory.length > 0;
|
|
890
|
+
|
|
891
|
+
// Send initial "thinking" indicator
|
|
892
|
+
if (chatWindow) {
|
|
893
|
+
chatWindow.webContents.send('agent-typing', { isTyping: true });
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
try {
|
|
897
|
+
// Call AI service
|
|
898
|
+
const result = await aiService.sendMessage(message, {
|
|
899
|
+
includeVisualContext
|
|
900
|
+
});
|
|
901
|
+
|
|
902
|
+
if (chatWindow) {
|
|
903
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
904
|
+
|
|
905
|
+
if (result.success) {
|
|
906
|
+
// Check if response contains actions
|
|
907
|
+
console.log('[AGENTIC] Parsing response for actions...');
|
|
908
|
+
const actionData = aiService.parseActions(result.message);
|
|
909
|
+
console.log('[AGENTIC] parseActions result:', actionData ? 'found' : 'null');
|
|
910
|
+
|
|
911
|
+
if (actionData && actionData.actions && actionData.actions.length > 0) {
|
|
912
|
+
console.log('[AGENTIC] AI returned actions:', actionData.actions.length);
|
|
913
|
+
console.log('[AGENTIC] Actions:', JSON.stringify(actionData.actions));
|
|
914
|
+
|
|
915
|
+
// Store pending actions
|
|
916
|
+
pendingActions = actionData;
|
|
917
|
+
|
|
918
|
+
// Send response with action data
|
|
919
|
+
chatWindow.webContents.send('agent-response', {
|
|
920
|
+
text: result.message,
|
|
921
|
+
timestamp: Date.now(),
|
|
922
|
+
provider: result.provider,
|
|
923
|
+
hasVisualContext: result.hasVisualContext,
|
|
924
|
+
hasActions: true,
|
|
925
|
+
actionData: actionData
|
|
926
|
+
});
|
|
927
|
+
|
|
928
|
+
// If agentic mode, execute immediately
|
|
929
|
+
if (agenticMode) {
|
|
930
|
+
console.log('[AGENTIC] Auto-executing actions (agentic mode)');
|
|
931
|
+
executeActionsAndRespond(actionData);
|
|
932
|
+
}
|
|
933
|
+
} else {
|
|
934
|
+
console.log('[AGENTIC] No actions detected in response');
|
|
935
|
+
// Normal response without actions
|
|
936
|
+
chatWindow.webContents.send('agent-response', {
|
|
937
|
+
text: result.message,
|
|
938
|
+
timestamp: Date.now(),
|
|
939
|
+
provider: result.provider,
|
|
940
|
+
hasVisualContext: result.hasVisualContext
|
|
941
|
+
});
|
|
942
|
+
}
|
|
943
|
+
} else {
|
|
944
|
+
chatWindow.webContents.send('agent-response', {
|
|
945
|
+
text: `Error: ${result.error}`,
|
|
946
|
+
type: 'error',
|
|
947
|
+
timestamp: Date.now()
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
} catch (error) {
|
|
952
|
+
console.error('AI service error:', error);
|
|
953
|
+
if (chatWindow) {
|
|
954
|
+
chatWindow.webContents.send('agent-typing', { isTyping: false });
|
|
955
|
+
chatWindow.webContents.send('agent-response', {
|
|
956
|
+
text: `Error: ${error.message}`,
|
|
957
|
+
type: 'error',
|
|
958
|
+
timestamp: Date.now()
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
});
|
|
963
|
+
|
|
964
|
+
// Helper for executing actions with visual feedback and overlay management
|
|
965
|
+
async function performSafeAgenticAction(action) {
|
|
966
|
+
// Only intercept clicks/drags that need overlay interaction
|
|
967
|
+
if (action.type === 'click' || action.type === 'double_click' || action.type === 'right_click' || action.type === 'drag') {
|
|
968
|
+
let x = action.x || action.fromX;
|
|
969
|
+
let y = action.y || action.fromY;
|
|
970
|
+
|
|
971
|
+
// Coordinate Scaling for Precision (Fix for Q4)
|
|
972
|
+
// If visual context exists, scale from Image Space -> Screen Space
|
|
973
|
+
const latestVisual = aiService.getLatestVisualContext();
|
|
974
|
+
if (latestVisual && latestVisual.width && latestVisual.height) {
|
|
975
|
+
const display = screen.getPrimaryDisplay();
|
|
976
|
+
const screenW = display.bounds.width; // e.g., 1920
|
|
977
|
+
const screenH = display.bounds.height; // e.g., 1080
|
|
978
|
+
// Calculate scale multiples
|
|
979
|
+
const scaleX = screenW / latestVisual.width;
|
|
980
|
+
const scaleY = screenH / latestVisual.height;
|
|
981
|
+
|
|
982
|
+
// Only apply if there's a significant difference (e.g. > 1% mismatch)
|
|
983
|
+
if (Math.abs(scaleX - 1) > 0.01 || Math.abs(scaleY - 1) > 0.01) {
|
|
984
|
+
console.log(`[EXECUTOR] Scaling coords from ${latestVisual.width}x${latestVisual.height} to ${screenW}x${screenH} (Target: ${x},${y})`);
|
|
985
|
+
x = Math.round(x * scaleX);
|
|
986
|
+
y = Math.round(y * scaleY);
|
|
987
|
+
// Update action object for system automation
|
|
988
|
+
if(action.x) action.x = x;
|
|
989
|
+
if(action.y) action.y = y;
|
|
990
|
+
if(action.fromX) action.fromX = x;
|
|
991
|
+
if(action.fromY) action.fromY = y;
|
|
992
|
+
if(action.toX) action.toX = Math.round(action.toX * scaleX);
|
|
993
|
+
if(action.toY) action.toY = Math.round(action.toY * scaleY);
|
|
994
|
+
console.log(`[EXECUTOR] Scaled target: ${x},${y}`);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
console.log(`[EXECUTOR] Intercepting ${action.type} at (${x},${y})`);
|
|
999
|
+
|
|
1000
|
+
// 1. Visual Feedback (Pulse - Doppler Effect)
|
|
1001
|
+
if (overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.webContents) {
|
|
1002
|
+
overlayWindow.webContents.send('overlay-command', {
|
|
1003
|
+
action: 'pulse-click',
|
|
1004
|
+
x: x,
|
|
1005
|
+
y: y,
|
|
1006
|
+
label: action.reason ? 'Action' : undefined
|
|
1007
|
+
});
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
// 2. Wait for user to see pulse (Doppler expansion)
|
|
1011
|
+
await new Promise(r => setTimeout(r, 600));
|
|
1012
|
+
|
|
1013
|
+
// 3. Prepare for Pass-through
|
|
1014
|
+
const wasVisible = overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible();
|
|
1015
|
+
if (wasVisible) {
|
|
1016
|
+
// A. Disable renderer pointer-events (CSS level)
|
|
1017
|
+
// This ensures elements like dots don't capture the click
|
|
1018
|
+
overlayWindow.webContents.send('overlay-command', {
|
|
1019
|
+
action: 'set-click-through',
|
|
1020
|
+
enabled: true
|
|
1021
|
+
});
|
|
1022
|
+
|
|
1023
|
+
// B. Set Electron window to ignore mouse events FULLY (no forwarding)
|
|
1024
|
+
// This ensures the window is completely transparent to the OS mouse subsystem
|
|
1025
|
+
overlayWindow.setIgnoreMouseEvents(true);
|
|
1026
|
+
|
|
1027
|
+
// Give OS time to update window regions
|
|
1028
|
+
await new Promise(r => setTimeout(r, 50));
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
// 4. Exec via System Automation
|
|
1032
|
+
let result;
|
|
1033
|
+
try {
|
|
1034
|
+
result = await aiService.systemAutomation.executeAction(action);
|
|
1035
|
+
} catch (e) {
|
|
1036
|
+
result = { success: false, error: e.message };
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
// 5. Restore Overlay Interactability
|
|
1040
|
+
if (wasVisible && overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1041
|
+
// Brief delay to ensure OS processed the click
|
|
1042
|
+
await new Promise(r => setTimeout(r, 50));
|
|
1043
|
+
|
|
1044
|
+
// A. Restore renderer pointer-events
|
|
1045
|
+
overlayWindow.webContents.send('overlay-command', {
|
|
1046
|
+
action: 'set-click-through',
|
|
1047
|
+
enabled: false
|
|
1048
|
+
});
|
|
1049
|
+
|
|
1050
|
+
// B. Restore Electron window behavior (forwarding enabled for UI interaction)
|
|
1051
|
+
// Note: We use forward: true so users can click dots but see through transparent areas
|
|
1052
|
+
overlayWindow.setIgnoreMouseEvents(true, { forward: true });
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
return result;
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
// Non-spatial actions (type, key, wait) - just execute
|
|
1059
|
+
return aiService.systemAutomation.executeAction(action);
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
// Execute actions and send results
|
|
1063
|
+
async function executeActionsAndRespond(actionData) {
|
|
1064
|
+
if (!chatWindow) return;
|
|
1065
|
+
|
|
1066
|
+
chatWindow.webContents.send('action-executing', {
|
|
1067
|
+
thought: actionData.thought,
|
|
1068
|
+
total: actionData.actions.length
|
|
1069
|
+
});
|
|
1070
|
+
|
|
1071
|
+
try {
|
|
1072
|
+
const results = await aiService.executeActions(
|
|
1073
|
+
actionData,
|
|
1074
|
+
// Progress callback
|
|
1075
|
+
(result, index, total) => {
|
|
1076
|
+
chatWindow.webContents.send('action-progress', {
|
|
1077
|
+
current: index + 1,
|
|
1078
|
+
total,
|
|
1079
|
+
result
|
|
1080
|
+
});
|
|
1081
|
+
},
|
|
1082
|
+
// Screenshot callback - MUST hide overlay before capture
|
|
1083
|
+
async () => {
|
|
1084
|
+
// Hide overlay before capturing so AI sees actual screen
|
|
1085
|
+
const wasOverlayVisible = overlayWindow && overlayWindow.isVisible();
|
|
1086
|
+
if (wasOverlayVisible) {
|
|
1087
|
+
overlayWindow.hide();
|
|
1088
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
const sources = await require('electron').desktopCapturer.getSources({
|
|
1092
|
+
types: ['screen'],
|
|
1093
|
+
thumbnailSize: {
|
|
1094
|
+
width: screen.getPrimaryDisplay().bounds.width,
|
|
1095
|
+
height: screen.getPrimaryDisplay().bounds.height
|
|
1096
|
+
}
|
|
1097
|
+
});
|
|
1098
|
+
|
|
1099
|
+
// Restore overlay after capture
|
|
1100
|
+
if (wasOverlayVisible && overlayWindow) {
|
|
1101
|
+
overlayWindow.show();
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
if (sources.length > 0) {
|
|
1105
|
+
const imageData = {
|
|
1106
|
+
dataURL: sources[0].thumbnail.toDataURL(),
|
|
1107
|
+
width: sources[0].thumbnail.getSize().width,
|
|
1108
|
+
height: sources[0].thumbnail.getSize().height,
|
|
1109
|
+
timestamp: Date.now()
|
|
1110
|
+
};
|
|
1111
|
+
storeVisualContext(imageData);
|
|
1112
|
+
}
|
|
1113
|
+
},
|
|
1114
|
+
// Options with safe executor
|
|
1115
|
+
{ actionExecutor: performSafeAgenticAction }
|
|
1116
|
+
);
|
|
1117
|
+
|
|
1118
|
+
// Send completion notification
|
|
1119
|
+
chatWindow.webContents.send('action-complete', {
|
|
1120
|
+
success: results.success,
|
|
1121
|
+
actionsCount: actionData.actions.length,
|
|
1122
|
+
thought: results.thought,
|
|
1123
|
+
verification: results.verification,
|
|
1124
|
+
results: results.results
|
|
1125
|
+
});
|
|
1126
|
+
|
|
1127
|
+
// If screenshot was requested, capture and show result
|
|
1128
|
+
if (results.screenshotRequested) {
|
|
1129
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
1130
|
+
|
|
1131
|
+
// Hide overlay before capturing
|
|
1132
|
+
const wasOverlayVisible = overlayWindow && overlayWindow.isVisible();
|
|
1133
|
+
if (wasOverlayVisible) {
|
|
1134
|
+
overlayWindow.hide();
|
|
1135
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
const sources = await require('electron').desktopCapturer.getSources({
|
|
1139
|
+
types: ['screen'],
|
|
1140
|
+
thumbnailSize: {
|
|
1141
|
+
width: screen.getPrimaryDisplay().bounds.width,
|
|
1142
|
+
height: screen.getPrimaryDisplay().bounds.height
|
|
1143
|
+
}
|
|
1144
|
+
});
|
|
1145
|
+
|
|
1146
|
+
// Restore overlay after capture
|
|
1147
|
+
if (wasOverlayVisible && overlayWindow) {
|
|
1148
|
+
overlayWindow.show();
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
if (sources.length > 0) {
|
|
1152
|
+
const imageData = {
|
|
1153
|
+
dataURL: sources[0].thumbnail.toDataURL(),
|
|
1154
|
+
width: sources[0].thumbnail.getSize().width,
|
|
1155
|
+
height: sources[0].thumbnail.getSize().height,
|
|
1156
|
+
timestamp: Date.now()
|
|
1157
|
+
};
|
|
1158
|
+
storeVisualContext(imageData);
|
|
1159
|
+
chatWindow.webContents.send('screen-captured', imageData);
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
} catch (error) {
|
|
1164
|
+
console.error('[AGENTIC] Action execution error:', error);
|
|
1165
|
+
chatWindow.webContents.send('action-complete', {
|
|
1166
|
+
success: false,
|
|
1167
|
+
actionsCount: actionData.actions ? actionData.actions.length : 0,
|
|
1168
|
+
error: error.message
|
|
1169
|
+
});
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
pendingActions = null;
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
// Handle confirmed action execution
|
|
1176
|
+
ipcMain.on('execute-actions', async (event, actionData) => {
|
|
1177
|
+
console.log('[AGENTIC] User confirmed action execution');
|
|
1178
|
+
await executeActionsAndRespond(actionData || pendingActions);
|
|
1179
|
+
});
|
|
1180
|
+
|
|
1181
|
+
// Handle action cancellation
|
|
1182
|
+
ipcMain.on('cancel-actions', () => {
|
|
1183
|
+
console.log('[AGENTIC] User cancelled actions');
|
|
1184
|
+
pendingActions = null;
|
|
1185
|
+
aiService.clearPendingAction();
|
|
1186
|
+
if (chatWindow) {
|
|
1187
|
+
chatWindow.webContents.send('agent-response', {
|
|
1188
|
+
text: 'Actions cancelled.',
|
|
1189
|
+
type: 'system',
|
|
1190
|
+
timestamp: Date.now()
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
});
|
|
1194
|
+
|
|
1195
|
+
// ===== SAFETY GUARDRAILS IPC HANDLERS =====
|
|
1196
|
+
|
|
1197
|
+
// Analyze action safety before execution
|
|
1198
|
+
ipcMain.handle('analyze-action-safety', (event, { action, targetInfo }) => {
|
|
1199
|
+
return aiService.analyzeActionSafety(action, targetInfo || {});
|
|
1200
|
+
});
|
|
1201
|
+
|
|
1202
|
+
// Get pending action awaiting confirmation
|
|
1203
|
+
ipcMain.handle('get-pending-action', () => {
|
|
1204
|
+
return aiService.getPendingAction();
|
|
1205
|
+
});
|
|
1206
|
+
|
|
1207
|
+
// Confirm pending action and resume execution
|
|
1208
|
+
ipcMain.handle('confirm-pending-action', async (event, { actionId }) => {
|
|
1209
|
+
console.log('[SAFETY] User confirmed action:', actionId);
|
|
1210
|
+
|
|
1211
|
+
const pending = aiService.getPendingAction();
|
|
1212
|
+
if (!pending || pending.actionId !== actionId) {
|
|
1213
|
+
return { success: false, error: 'No matching pending action' };
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
// Resume execution after confirmation
|
|
1217
|
+
try {
|
|
1218
|
+
const results = await aiService.resumeAfterConfirmation(
|
|
1219
|
+
// Progress callback
|
|
1220
|
+
(result, index, total) => {
|
|
1221
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1222
|
+
chatWindow.webContents.send('action-progress', {
|
|
1223
|
+
current: index + 1,
|
|
1224
|
+
total,
|
|
1225
|
+
result,
|
|
1226
|
+
userConfirmed: true
|
|
1227
|
+
});
|
|
1228
|
+
}
|
|
1229
|
+
},
|
|
1230
|
+
// Screenshot callback
|
|
1231
|
+
async () => {
|
|
1232
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1233
|
+
overlayWindow.hide();
|
|
1234
|
+
}
|
|
1235
|
+
await new Promise(r => setTimeout(r, 100));
|
|
1236
|
+
|
|
1237
|
+
const sources = await desktopCapturer.getSources({
|
|
1238
|
+
types: ['screen'],
|
|
1239
|
+
thumbnailSize: {
|
|
1240
|
+
width: screen.getPrimaryDisplay().bounds.width,
|
|
1241
|
+
height: screen.getPrimaryDisplay().bounds.height
|
|
1242
|
+
}
|
|
1243
|
+
});
|
|
1244
|
+
|
|
1245
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1246
|
+
overlayWindow.show();
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
if (sources.length > 0) {
|
|
1250
|
+
const imageData = {
|
|
1251
|
+
dataURL: sources[0].thumbnail.toDataURL(),
|
|
1252
|
+
width: sources[0].thumbnail.getSize().width,
|
|
1253
|
+
height: sources[0].thumbnail.getSize().height,
|
|
1254
|
+
timestamp: Date.now()
|
|
1255
|
+
};
|
|
1256
|
+
storeVisualContext(imageData);
|
|
1257
|
+
}
|
|
1258
|
+
},
|
|
1259
|
+
// Options with safe executor
|
|
1260
|
+
{ actionExecutor: performSafeAgenticAction }
|
|
1261
|
+
);
|
|
1262
|
+
|
|
1263
|
+
// Notify chat of completion
|
|
1264
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1265
|
+
chatWindow.webContents.send('action-complete', {
|
|
1266
|
+
success: results.success,
|
|
1267
|
+
userConfirmed: true,
|
|
1268
|
+
results: results.results
|
|
1269
|
+
});
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
return { success: true, results };
|
|
1273
|
+
} catch (error) {
|
|
1274
|
+
console.error('[SAFETY] Resume after confirmation failed:', error);
|
|
1275
|
+
return { success: false, error: error.message };
|
|
1276
|
+
}
|
|
1277
|
+
});
|
|
1278
|
+
|
|
1279
|
+
// Reject pending action
|
|
1280
|
+
ipcMain.handle('reject-pending-action', (event, { actionId }) => {
|
|
1281
|
+
console.log('[SAFETY] User rejected action:', actionId);
|
|
1282
|
+
|
|
1283
|
+
const rejected = aiService.rejectPendingAction(actionId);
|
|
1284
|
+
|
|
1285
|
+
if (rejected && chatWindow && !chatWindow.isDestroyed()) {
|
|
1286
|
+
chatWindow.webContents.send('action-rejected', {
|
|
1287
|
+
actionId,
|
|
1288
|
+
message: 'Action rejected by user'
|
|
1289
|
+
});
|
|
1290
|
+
chatWindow.webContents.send('agent-response', {
|
|
1291
|
+
text: '🛡️ Action rejected. The potentially risky action was not executed.',
|
|
1292
|
+
type: 'system',
|
|
1293
|
+
timestamp: Date.now()
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
return { success: rejected };
|
|
1298
|
+
});
|
|
1299
|
+
|
|
1300
|
+
// Convert grid label to screen coordinates
|
|
1301
|
+
ipcMain.handle('label-to-coordinates', (event, label) => {
|
|
1302
|
+
// Use gridToPixels from ai-service which uses system-automation
|
|
1303
|
+
const coords = aiService.gridToPixels(label);
|
|
1304
|
+
if (coords) {
|
|
1305
|
+
return {
|
|
1306
|
+
success: true,
|
|
1307
|
+
label,
|
|
1308
|
+
x: coords.x,
|
|
1309
|
+
y: coords.y,
|
|
1310
|
+
screenX: coords.x,
|
|
1311
|
+
screenY: coords.y
|
|
1312
|
+
};
|
|
1313
|
+
}
|
|
1314
|
+
return { success: false, error: `Invalid grid label: ${label}` };
|
|
1315
|
+
});
|
|
1316
|
+
|
|
1317
|
+
// Safe click with overlay hide/show and safety analysis
|
|
1318
|
+
ipcMain.handle('safe-click-at', async (event, { x, y, button = 'left', label, targetInfo }) => {
|
|
1319
|
+
console.log(`[SAFETY] Safe click requested at (${x}, ${y}), button: ${button}`);
|
|
1320
|
+
|
|
1321
|
+
// Analyze safety
|
|
1322
|
+
const action = { type: 'click', x, y, button, reason: label || '' };
|
|
1323
|
+
const safety = aiService.analyzeActionSafety(action, targetInfo || {});
|
|
1324
|
+
|
|
1325
|
+
// If HIGH or CRITICAL, don't execute - require explicit confirmation
|
|
1326
|
+
if (safety.requiresConfirmation) {
|
|
1327
|
+
console.log(`[SAFETY] Click requires confirmation: ${safety.riskLevel}`);
|
|
1328
|
+
|
|
1329
|
+
aiService.setPendingAction({
|
|
1330
|
+
...safety,
|
|
1331
|
+
actionIndex: 0,
|
|
1332
|
+
remainingActions: [action],
|
|
1333
|
+
completedResults: [],
|
|
1334
|
+
thought: `Click at (${x}, ${y})`,
|
|
1335
|
+
verification: 'Verify click target'
|
|
1336
|
+
});
|
|
1337
|
+
|
|
1338
|
+
// Notify chat window
|
|
1339
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1340
|
+
chatWindow.webContents.send('action-requires-confirmation', {
|
|
1341
|
+
actionId: safety.actionId,
|
|
1342
|
+
action: action,
|
|
1343
|
+
safety: safety,
|
|
1344
|
+
description: safety.description,
|
|
1345
|
+
riskLevel: safety.riskLevel,
|
|
1346
|
+
warnings: safety.warnings
|
|
1347
|
+
});
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
return {
|
|
1351
|
+
success: false,
|
|
1352
|
+
pending: true,
|
|
1353
|
+
actionId: safety.actionId,
|
|
1354
|
+
riskLevel: safety.riskLevel,
|
|
1355
|
+
message: `Action requires confirmation: ${safety.warnings.join(', ')}`
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
// SAFE/LOW/MEDIUM - execute with visual feedback
|
|
1360
|
+
try {
|
|
1361
|
+
// INJECTION: Ensure visual feedback system is loaded
|
|
1362
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1363
|
+
try {
|
|
1364
|
+
const isLoaded = await overlayWindow.webContents.executeJavaScript('window.hasPulseSystem === true').catch(() => false);
|
|
1365
|
+
|
|
1366
|
+
if (!isLoaded) {
|
|
1367
|
+
const css = `
|
|
1368
|
+
.pulse-ring {
|
|
1369
|
+
position: absolute;
|
|
1370
|
+
border-radius: 50%;
|
|
1371
|
+
pointer-events: none;
|
|
1372
|
+
animation: pulse-animation 0.8s ease-out forwards;
|
|
1373
|
+
border: 2px solid #00ffcc;
|
|
1374
|
+
background: radial-gradient(circle, rgba(0,255,204,0.3) 0%, rgba(0,255,204,0) 70%);
|
|
1375
|
+
box-shadow: 0 0 15px rgba(0, 255, 204, 0.6);
|
|
1376
|
+
z-index: 2147483647;
|
|
1377
|
+
transform: translate(-50%, -50%);
|
|
1378
|
+
}
|
|
1379
|
+
@keyframes pulse-animation {
|
|
1380
|
+
0% { width: 10px; height: 10px; opacity: 1; transform: translate(-50%, -50%) scale(1); }
|
|
1381
|
+
100% { width: 100px; height: 100px; opacity: 0; transform: translate(-50%, -50%) scale(1.5); }
|
|
1382
|
+
}
|
|
1383
|
+
`;
|
|
1384
|
+
await overlayWindow.webContents.insertCSS(css);
|
|
1385
|
+
overlayWindow.webContents.executeJavaScript(`
|
|
1386
|
+
const { ipcRenderer } = require('electron');
|
|
1387
|
+
window.showPulseClick = (x, y) => {
|
|
1388
|
+
const el = document.createElement('div');
|
|
1389
|
+
el.className = 'pulse-ring';
|
|
1390
|
+
el.style.left = x + 'px';
|
|
1391
|
+
el.style.top = y + 'px';
|
|
1392
|
+
document.body.appendChild(el);
|
|
1393
|
+
setTimeout(() => el.remove(), 1000);
|
|
1394
|
+
};
|
|
1395
|
+
ipcRenderer.removeAllListeners('overlay-command');
|
|
1396
|
+
ipcRenderer.on('overlay-command', (event, data) => {
|
|
1397
|
+
if (data.action === 'pulse-click') window.showPulseClick(data.x, data.y);
|
|
1398
|
+
});
|
|
1399
|
+
window.hasPulseSystem = true;
|
|
1400
|
+
`);
|
|
1401
|
+
}
|
|
1402
|
+
} catch(e) { console.error('Safe click injection error:', e); }
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
// Show visual indicator on overlay
|
|
1406
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1407
|
+
overlayWindow.webContents.send('overlay-command', {
|
|
1408
|
+
action: 'pulse-click', // Updated to pulse
|
|
1409
|
+
x, y,
|
|
1410
|
+
label: label || `${x},${y}`
|
|
1411
|
+
});
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1415
|
+
|
|
1416
|
+
// Hide overlay for click-through
|
|
1417
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1418
|
+
overlayWindow.hide();
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
await new Promise(r => setTimeout(r, 50));
|
|
1422
|
+
|
|
1423
|
+
// Execute click via system-automation
|
|
1424
|
+
const result = await aiService.systemAutomation.executeAction({
|
|
1425
|
+
type: 'click',
|
|
1426
|
+
x: Math.round(x),
|
|
1427
|
+
y: Math.round(y),
|
|
1428
|
+
button
|
|
1429
|
+
});
|
|
1430
|
+
|
|
1431
|
+
await new Promise(r => setTimeout(r, 100));
|
|
1432
|
+
|
|
1433
|
+
// Restore overlay
|
|
1434
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1435
|
+
overlayWindow.show();
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
console.log(`[SAFETY] Click executed: ${result.success}`);
|
|
1439
|
+
|
|
1440
|
+
return {
|
|
1441
|
+
success: result.success,
|
|
1442
|
+
x, y,
|
|
1443
|
+
riskLevel: safety.riskLevel,
|
|
1444
|
+
error: result.error
|
|
1445
|
+
};
|
|
1446
|
+
|
|
1447
|
+
} catch (error) {
|
|
1448
|
+
console.error('[SAFETY] Safe click failed:', error);
|
|
1449
|
+
|
|
1450
|
+
// Always restore overlay on error
|
|
1451
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1452
|
+
overlayWindow.show();
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
return { success: false, error: error.message };
|
|
1456
|
+
}
|
|
1457
|
+
});
|
|
1458
|
+
|
|
1459
|
+
// ===== WINDOW CONTROLS =====
|
|
1460
|
+
ipcMain.on('minimize-chat', () => {
|
|
1461
|
+
if (chatWindow) {
|
|
1462
|
+
chatWindow.minimize();
|
|
1463
|
+
}
|
|
1464
|
+
});
|
|
1465
|
+
|
|
1466
|
+
ipcMain.on('hide-chat', () => {
|
|
1467
|
+
if (chatWindow) {
|
|
1468
|
+
chatWindow.hide();
|
|
1469
|
+
isChatVisible = false;
|
|
1470
|
+
}
|
|
1471
|
+
});
|
|
1472
|
+
|
|
1473
|
+
// ===== SCREEN CAPTURE (AI Visual Awareness) =====
|
|
1474
|
+
// CRITICAL: Hide overlay before capture so AI sees actual screen content without dots
|
|
1475
|
+
ipcMain.on('capture-screen', async (event, options = {}) => {
|
|
1476
|
+
try {
|
|
1477
|
+
// Hide overlay BEFORE capturing so screenshot shows actual screen (not dots)
|
|
1478
|
+
const wasOverlayVisible = overlayWindow && overlayWindow.isVisible();
|
|
1479
|
+
if (wasOverlayVisible) {
|
|
1480
|
+
overlayWindow.hide();
|
|
1481
|
+
// Brief delay to ensure overlay is fully hidden
|
|
1482
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
const sources = await desktopCapturer.getSources({
|
|
1486
|
+
types: ['screen'],
|
|
1487
|
+
thumbnailSize: {
|
|
1488
|
+
width: screen.getPrimaryDisplay().bounds.width,
|
|
1489
|
+
height: screen.getPrimaryDisplay().bounds.height
|
|
1490
|
+
}
|
|
1491
|
+
});
|
|
1492
|
+
|
|
1493
|
+
// Restore overlay after capture
|
|
1494
|
+
if (wasOverlayVisible && overlayWindow) {
|
|
1495
|
+
overlayWindow.show();
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
if (sources.length > 0) {
|
|
1499
|
+
const primarySource = sources[0];
|
|
1500
|
+
const thumbnail = primarySource.thumbnail;
|
|
1501
|
+
|
|
1502
|
+
// Get image data
|
|
1503
|
+
const imageData = {
|
|
1504
|
+
dataURL: thumbnail.toDataURL(),
|
|
1505
|
+
width: thumbnail.getSize().width,
|
|
1506
|
+
height: thumbnail.getSize().height,
|
|
1507
|
+
x: 0,
|
|
1508
|
+
y: 0,
|
|
1509
|
+
timestamp: Date.now(),
|
|
1510
|
+
sourceId: primarySource.id,
|
|
1511
|
+
sourceName: primarySource.name
|
|
1512
|
+
};
|
|
1513
|
+
|
|
1514
|
+
// Send to chat window
|
|
1515
|
+
if (chatWindow) {
|
|
1516
|
+
chatWindow.webContents.send('screen-captured', imageData);
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
// Log for debugging
|
|
1520
|
+
console.log(`Screen captured: ${imageData.width}x${imageData.height} (overlay was ${wasOverlayVisible ? 'hidden' : 'already hidden'})`);
|
|
1521
|
+
|
|
1522
|
+
// Store in visual context for AI processing
|
|
1523
|
+
storeVisualContext(imageData);
|
|
1524
|
+
}
|
|
1525
|
+
} catch (error) {
|
|
1526
|
+
console.error('Screen capture failed:', error);
|
|
1527
|
+
// Ensure overlay is restored on error
|
|
1528
|
+
if (overlayWindow && !overlayWindow.isVisible()) {
|
|
1529
|
+
overlayWindow.show();
|
|
1530
|
+
}
|
|
1531
|
+
if (chatWindow) {
|
|
1532
|
+
chatWindow.webContents.send('screen-captured', { error: error.message });
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
});
|
|
1536
|
+
|
|
1537
|
+
// Capture a specific region
|
|
1538
|
+
ipcMain.on('capture-region', async (event, { x, y, width, height }) => {
|
|
1539
|
+
try {
|
|
1540
|
+
// Hide overlay BEFORE capturing
|
|
1541
|
+
const wasOverlayVisible = overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible();
|
|
1542
|
+
if (wasOverlayVisible) {
|
|
1543
|
+
overlayWindow.hide();
|
|
1544
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
const sources = await desktopCapturer.getSources({
|
|
1548
|
+
types: ['screen'],
|
|
1549
|
+
thumbnailSize: {
|
|
1550
|
+
width: screen.getPrimaryDisplay().bounds.width,
|
|
1551
|
+
height: screen.getPrimaryDisplay().bounds.height
|
|
1552
|
+
}
|
|
1553
|
+
});
|
|
1554
|
+
|
|
1555
|
+
// Restore overlay after capture
|
|
1556
|
+
if (wasOverlayVisible && overlayWindow) {
|
|
1557
|
+
overlayWindow.show();
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
if (sources.length > 0) {
|
|
1561
|
+
const primarySource = sources[0];
|
|
1562
|
+
const thumbnail = primarySource.thumbnail;
|
|
1563
|
+
|
|
1564
|
+
// Crop to region
|
|
1565
|
+
const cropped = thumbnail.crop({
|
|
1566
|
+
x: Math.max(0, x),
|
|
1567
|
+
y: Math.max(0, y),
|
|
1568
|
+
width: Math.min(width, thumbnail.getSize().width - x),
|
|
1569
|
+
height: Math.min(height, thumbnail.getSize().height - y)
|
|
1570
|
+
});
|
|
1571
|
+
|
|
1572
|
+
const imageData = {
|
|
1573
|
+
dataURL: cropped.toDataURL(),
|
|
1574
|
+
width: cropped.getSize().width,
|
|
1575
|
+
height: cropped.getSize().height,
|
|
1576
|
+
x,
|
|
1577
|
+
y,
|
|
1578
|
+
timestamp: Date.now(),
|
|
1579
|
+
type: 'region'
|
|
1580
|
+
};
|
|
1581
|
+
|
|
1582
|
+
if (chatWindow) {
|
|
1583
|
+
chatWindow.webContents.send('screen-captured', imageData);
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
storeVisualContext(imageData);
|
|
1587
|
+
}
|
|
1588
|
+
} catch (error) {
|
|
1589
|
+
console.error('Region capture failed:', error);
|
|
1590
|
+
// Ensure overlay is restored on error
|
|
1591
|
+
if (overlayWindow && !overlayWindow.isVisible()) {
|
|
1592
|
+
overlayWindow.show();
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
});
|
|
1596
|
+
|
|
1597
|
+
// Get current state
|
|
1598
|
+
ipcMain.handle('get-state', () => {
|
|
1599
|
+
const aiStatus = aiService.getStatus();
|
|
1600
|
+
return {
|
|
1601
|
+
overlayMode,
|
|
1602
|
+
isChatVisible,
|
|
1603
|
+
visualContextCount: visualContextHistory.length,
|
|
1604
|
+
aiProvider: aiStatus.provider,
|
|
1605
|
+
model: aiStatus.model,
|
|
1606
|
+
aiStatus,
|
|
1607
|
+
// Inspect mode state
|
|
1608
|
+
inspectMode: inspectService.isInspectModeActive(),
|
|
1609
|
+
inspectRegionCount: inspectService.getRegions().length
|
|
1610
|
+
};
|
|
1611
|
+
});
|
|
1612
|
+
|
|
1613
|
+
// ===== INSPECT MODE IPC HANDLERS =====
|
|
1614
|
+
|
|
1615
|
+
// Toggle inspect mode
|
|
1616
|
+
ipcMain.on('toggle-inspect-mode', () => {
|
|
1617
|
+
const newState = !inspectService.isInspectModeActive();
|
|
1618
|
+
inspectService.setInspectMode(newState);
|
|
1619
|
+
console.log(`[INSPECT] Mode toggled: ${newState}`);
|
|
1620
|
+
|
|
1621
|
+
// Notify overlay
|
|
1622
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1623
|
+
overlayWindow.webContents.send('inspect-mode-changed', newState);
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
// Notify chat
|
|
1627
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1628
|
+
chatWindow.webContents.send('inspect-mode-changed', newState);
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
// If enabled, trigger region detection
|
|
1632
|
+
if (newState) {
|
|
1633
|
+
detectAndSendInspectRegions().catch(err => {
|
|
1634
|
+
console.error('[INSPECT] Region detection failed:', err);
|
|
1635
|
+
});
|
|
1636
|
+
}
|
|
1637
|
+
});
|
|
1638
|
+
|
|
1639
|
+
// Request inspect regions detection
|
|
1640
|
+
ipcMain.on('request-inspect-regions', async () => {
|
|
1641
|
+
await detectAndSendInspectRegions().catch(err => {
|
|
1642
|
+
console.error('[INSPECT] Region detection request failed:', err);
|
|
1643
|
+
});
|
|
1644
|
+
});
|
|
1645
|
+
|
|
1646
|
+
// Handle inspect region selection from overlay
|
|
1647
|
+
ipcMain.on('inspect-region-selected', (event, data) => {
|
|
1648
|
+
console.log('[INSPECT] Region selected:', data);
|
|
1649
|
+
|
|
1650
|
+
// Record the action
|
|
1651
|
+
const trace = inspectService.recordAction({
|
|
1652
|
+
type: 'select',
|
|
1653
|
+
targetId: data.targetId,
|
|
1654
|
+
x: data.x,
|
|
1655
|
+
y: data.y
|
|
1656
|
+
}, data.targetId);
|
|
1657
|
+
|
|
1658
|
+
// Forward to chat window with targetId for AI targeting
|
|
1659
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1660
|
+
chatWindow.webContents.send('inspect-region-selected', {
|
|
1661
|
+
...data,
|
|
1662
|
+
actionId: trace.actionId
|
|
1663
|
+
});
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
// Select the region in service
|
|
1667
|
+
inspectService.selectRegion(data.targetId);
|
|
1668
|
+
});
|
|
1669
|
+
|
|
1670
|
+
// Get inspect context for AI
|
|
1671
|
+
ipcMain.handle('get-inspect-context', () => {
|
|
1672
|
+
return inspectService.generateAIContext();
|
|
1673
|
+
});
|
|
1674
|
+
|
|
1675
|
+
// Get inspect regions
|
|
1676
|
+
ipcMain.handle('get-inspect-regions', () => {
|
|
1677
|
+
return inspectService.getRegions();
|
|
1678
|
+
});
|
|
1679
|
+
|
|
1680
|
+
// Get window context
|
|
1681
|
+
ipcMain.handle('get-window-context', async () => {
|
|
1682
|
+
return await inspectService.updateWindowContext();
|
|
1683
|
+
});
|
|
1684
|
+
|
|
1685
|
+
/**
|
|
1686
|
+
* Detect UI regions and send to overlay
|
|
1687
|
+
*/
|
|
1688
|
+
async function detectAndSendInspectRegions() {
|
|
1689
|
+
try {
|
|
1690
|
+
console.log('[INSPECT] Detecting regions...');
|
|
1691
|
+
const results = await inspectService.detectRegions();
|
|
1692
|
+
|
|
1693
|
+
// Send regions to overlay
|
|
1694
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1695
|
+
overlayWindow.webContents.send('inspect-regions-update', results.regions);
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
// Notify chat of new context
|
|
1699
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1700
|
+
chatWindow.webContents.send('inspect-context-update', {
|
|
1701
|
+
regionCount: results.regions.length,
|
|
1702
|
+
windowContext: results.windowContext
|
|
1703
|
+
});
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
console.log(`[INSPECT] Detected ${results.regions.length} regions`);
|
|
1707
|
+
return results;
|
|
1708
|
+
} catch (error) {
|
|
1709
|
+
console.error('[INSPECT] Detection failed:', error);
|
|
1710
|
+
return { regions: [], error: error.message };
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
// ===== AI CLICK-THROUGH AUTOMATION (Q4 FIX) =====
|
|
1715
|
+
// This allows AI to click at coordinates THROUGH the overlay to the background app
|
|
1716
|
+
// The overlay should NOT intercept these programmatic clicks
|
|
1717
|
+
ipcMain.handle('click-through-at', async (event, { x, y, button = 'left', label }) => {
|
|
1718
|
+
try {
|
|
1719
|
+
console.log(`[CLICK-THROUGH] Executing click at (${x}, ${y}) label=${label || 'none'}`);
|
|
1720
|
+
|
|
1721
|
+
// INJECTION: Ensure visual feedback system is loaded on first click
|
|
1722
|
+
if (overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1723
|
+
try {
|
|
1724
|
+
// Check if pulse system is loaded in renderer
|
|
1725
|
+
const isLoaded = await overlayWindow.webContents.executeJavaScript('window.hasPulseSystem === true').catch(() => false);
|
|
1726
|
+
|
|
1727
|
+
if (!isLoaded) {
|
|
1728
|
+
console.log('[CLICK-THROUGH] Injecting visual feedback system...');
|
|
1729
|
+
const css = `
|
|
1730
|
+
.pulse-ring {
|
|
1731
|
+
position: absolute;
|
|
1732
|
+
border-radius: 50%;
|
|
1733
|
+
pointer-events: none;
|
|
1734
|
+
animation: pulse-animation 0.8s ease-out forwards;
|
|
1735
|
+
border: 2px solid #00ffcc;
|
|
1736
|
+
background: radial-gradient(circle, rgba(0,255,204,0.3) 0%, rgba(0,255,204,0) 70%);
|
|
1737
|
+
box-shadow: 0 0 15px rgba(0, 255, 204, 0.6);
|
|
1738
|
+
z-index: 2147483647;
|
|
1739
|
+
transform: translate(-50%, -50%);
|
|
1740
|
+
}
|
|
1741
|
+
@keyframes pulse-animation {
|
|
1742
|
+
0% { width: 10px; height: 10px; opacity: 1; transform: translate(-50%, -50%) scale(1); }
|
|
1743
|
+
100% { width: 100px; height: 100px; opacity: 0; transform: translate(-50%, -50%) scale(1.5); }
|
|
1744
|
+
}
|
|
1745
|
+
`;
|
|
1746
|
+
await overlayWindow.webContents.insertCSS(css);
|
|
1747
|
+
|
|
1748
|
+
const js = `
|
|
1749
|
+
const { ipcRenderer } = require('electron');
|
|
1750
|
+
window.showPulseClick = (x, y) => {
|
|
1751
|
+
const el = document.createElement('div');
|
|
1752
|
+
el.className = 'pulse-ring';
|
|
1753
|
+
el.style.left = x + 'px';
|
|
1754
|
+
el.style.top = y + 'px';
|
|
1755
|
+
document.body.appendChild(el);
|
|
1756
|
+
setTimeout(() => el.remove(), 1000);
|
|
1757
|
+
};
|
|
1758
|
+
ipcRenderer.removeAllListeners('overlay-command');
|
|
1759
|
+
ipcRenderer.on('overlay-command', (event, data) => {
|
|
1760
|
+
if (data.action === 'pulse-click') window.showPulseClick(data.x, data.y);
|
|
1761
|
+
});
|
|
1762
|
+
window.hasPulseSystem = true;
|
|
1763
|
+
`;
|
|
1764
|
+
await overlayWindow.webContents.executeJavaScript(js);
|
|
1765
|
+
}
|
|
1766
|
+
} catch (e) { console.error('Visual injection error:', e); }
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
// 1. Show visual feedback on overlay (optional - for user awareness)
|
|
1770
|
+
if (overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.webContents) {
|
|
1771
|
+
overlayWindow.webContents.send('overlay-command', {
|
|
1772
|
+
action: 'pulse-click', // Changed from highlight-coordinate to specific pulse-click
|
|
1773
|
+
x, y, label
|
|
1774
|
+
});
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
// 2. Brief delay for visual feedback (increased to let pulse show)
|
|
1778
|
+
await new Promise(resolve => setTimeout(resolve, 300));
|
|
1779
|
+
|
|
1780
|
+
// 3. Hide overlay to ensure click goes through
|
|
1781
|
+
const wasVisible = overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible();
|
|
1782
|
+
if (wasVisible) {
|
|
1783
|
+
overlayWindow.hide();
|
|
1784
|
+
// Give Windows DWM more time to process transparency
|
|
1785
|
+
await new Promise(resolve => setTimeout(resolve, 150));
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
// 4. Execute the click using robotjs or similar automation
|
|
1789
|
+
// Note: This requires robotjs to be installed and working
|
|
1790
|
+
try {
|
|
1791
|
+
const robot = require('robotjs');
|
|
1792
|
+
// Double move to ensure OS registers cursor position
|
|
1793
|
+
robot.moveMouse(x, y);
|
|
1794
|
+
robot.moveMouse(x, y);
|
|
1795
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
1796
|
+
robot.mouseClick(button);
|
|
1797
|
+
console.log(`[CLICK-THROUGH] Click executed successfully at (${x}, ${y})`);
|
|
1798
|
+
} catch (robotError) {
|
|
1799
|
+
console.error('[CLICK-THROUGH] Robot click failed:', robotError.message);
|
|
1800
|
+
// Fallback: try using PowerShell on Windows
|
|
1801
|
+
if (process.platform === 'win32') {
|
|
1802
|
+
const { exec } = require('child_process');
|
|
1803
|
+
const psCommand = `
|
|
1804
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
1805
|
+
[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})
|
|
1806
|
+
Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo);' -Name U32 -Namespace W
|
|
1807
|
+
[W.U32]::mouse_event(0x02, 0, 0, 0, 0)
|
|
1808
|
+
[W.U32]::mouse_event(0x04, 0, 0, 0, 0)
|
|
1809
|
+
`;
|
|
1810
|
+
await new Promise((resolve, reject) => {
|
|
1811
|
+
exec(`powershell -Command "${psCommand.replace(/"/g, '\\"')}"`, (error) => {
|
|
1812
|
+
if (error) reject(error);
|
|
1813
|
+
else resolve();
|
|
1814
|
+
});
|
|
1815
|
+
});
|
|
1816
|
+
console.log(`[CLICK-THROUGH] PowerShell click executed at (${x}, ${y})`);
|
|
1817
|
+
} else {
|
|
1818
|
+
throw robotError;
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
|
|
1822
|
+
// 5. Restore overlay after a delay (let the click register)
|
|
1823
|
+
await new Promise(resolve => setTimeout(resolve, 150));
|
|
1824
|
+
if (wasVisible && overlayWindow && !overlayWindow.isDestroyed()) {
|
|
1825
|
+
overlayWindow.show();
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
return { success: true, x, y, label };
|
|
1829
|
+
} catch (error) {
|
|
1830
|
+
console.error('[CLICK-THROUGH] Error:', error);
|
|
1831
|
+
// Ensure overlay is restored on error
|
|
1832
|
+
if (overlayWindow && !overlayWindow.isDestroyed() && !overlayWindow.isVisible()) {
|
|
1833
|
+
overlayWindow.show();
|
|
1834
|
+
}
|
|
1835
|
+
return { success: false, error: error.message };
|
|
1836
|
+
}
|
|
1837
|
+
});
|
|
1838
|
+
|
|
1839
|
+
// NOTE: label-to-coordinates, analyze-action-safety, safe-click-at, confirm-pending-action,
|
|
1840
|
+
// reject-pending-action, and get-pending-action handlers are registered above in
|
|
1841
|
+
// SAFETY GUARDRAILS IPC HANDLERS section. Do NOT register duplicate handlers here.
|
|
1842
|
+
|
|
1843
|
+
// NOTE: strict mode requires unique IPC handlers
|
|
1844
|
+
// Previously duplicate handlers were removed from here.
|
|
1845
|
+
|
|
1846
|
+
// Set AI provider
|
|
1847
|
+
ipcMain.on('set-ai-provider', (event, provider) => {
|
|
1848
|
+
const success = aiService.setProvider(provider);
|
|
1849
|
+
if (chatWindow) {
|
|
1850
|
+
chatWindow.webContents.send('provider-changed', {
|
|
1851
|
+
provider,
|
|
1852
|
+
success,
|
|
1853
|
+
status: aiService.getStatus()
|
|
1854
|
+
});
|
|
1855
|
+
}
|
|
1856
|
+
});
|
|
1857
|
+
|
|
1858
|
+
// Set API key
|
|
1859
|
+
ipcMain.on('set-api-key', (event, { provider, key }) => {
|
|
1860
|
+
const success = aiService.setApiKey(provider, key);
|
|
1861
|
+
if (chatWindow) {
|
|
1862
|
+
chatWindow.webContents.send('api-key-set', { provider, success });
|
|
1863
|
+
}
|
|
1864
|
+
});
|
|
1865
|
+
|
|
1866
|
+
// Check auth status for a provider
|
|
1867
|
+
ipcMain.on('check-auth', async (event, provider) => {
|
|
1868
|
+
const status = aiService.getStatus();
|
|
1869
|
+
const currentProvider = provider || status.provider;
|
|
1870
|
+
let authStatus = 'disconnected';
|
|
1871
|
+
|
|
1872
|
+
if (currentProvider === 'copilot') {
|
|
1873
|
+
// Check if Copilot token exists
|
|
1874
|
+
const tokenPath = require('path').join(app.getPath('appData'), 'copilot-agent', 'copilot-token.json');
|
|
1875
|
+
try {
|
|
1876
|
+
if (require('fs').existsSync(tokenPath)) {
|
|
1877
|
+
authStatus = 'connected';
|
|
1878
|
+
}
|
|
1879
|
+
} catch (e) {
|
|
1880
|
+
authStatus = 'disconnected';
|
|
1881
|
+
}
|
|
1882
|
+
} else if (currentProvider === 'ollama') {
|
|
1883
|
+
// Ollama doesn't need auth, just check if running
|
|
1884
|
+
authStatus = 'connected';
|
|
1885
|
+
} else {
|
|
1886
|
+
// OpenAI/Anthropic need API keys
|
|
1887
|
+
authStatus = status.hasApiKey ? 'connected' : 'disconnected';
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
if (chatWindow) {
|
|
1891
|
+
chatWindow.webContents.send('auth-status', {
|
|
1892
|
+
provider: currentProvider,
|
|
1893
|
+
status: authStatus
|
|
1894
|
+
});
|
|
1895
|
+
}
|
|
1896
|
+
});
|
|
1897
|
+
|
|
1898
|
+
// ===== VISUAL AWARENESS =====
|
|
1899
|
+
|
|
1900
|
+
// Get active window info
|
|
1901
|
+
ipcMain.handle('get-active-window', async () => {
|
|
1902
|
+
return await visualAwareness.getActiveWindow();
|
|
1903
|
+
});
|
|
1904
|
+
|
|
1905
|
+
// Find element at coordinates
|
|
1906
|
+
ipcMain.handle('find-element-at', async (event, { x, y }) => {
|
|
1907
|
+
return await visualAwareness.findElementAtPoint(x, y);
|
|
1908
|
+
});
|
|
1909
|
+
|
|
1910
|
+
// Detect UI elements
|
|
1911
|
+
ipcMain.handle('detect-ui-elements', async (event, options = {}) => {
|
|
1912
|
+
return await visualAwareness.detectUIElements(options);
|
|
1913
|
+
});
|
|
1914
|
+
|
|
1915
|
+
// Extract text via OCR
|
|
1916
|
+
ipcMain.handle('extract-text', async (event, options = {}) => {
|
|
1917
|
+
const latestContext = visualContextHistory[visualContextHistory.length - 1];
|
|
1918
|
+
if (!latestContext) {
|
|
1919
|
+
return { error: 'No screen capture available. Capture screen first.' };
|
|
1920
|
+
}
|
|
1921
|
+
return await visualAwareness.extractTextFromImage(latestContext, options);
|
|
1922
|
+
});
|
|
1923
|
+
|
|
1924
|
+
// Full screen analysis
|
|
1925
|
+
ipcMain.handle('analyze-screen', async (event, options = {}) => {
|
|
1926
|
+
const latestContext = visualContextHistory[visualContextHistory.length - 1];
|
|
1927
|
+
if (!latestContext) {
|
|
1928
|
+
return { error: 'No screen capture available. Capture screen first.' };
|
|
1929
|
+
}
|
|
1930
|
+
const analysis = await visualAwareness.analyzeScreen(latestContext, options);
|
|
1931
|
+
|
|
1932
|
+
// Send analysis to chat window
|
|
1933
|
+
if (chatWindow) {
|
|
1934
|
+
chatWindow.webContents.send('screen-analysis', analysis);
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
return analysis;
|
|
1938
|
+
});
|
|
1939
|
+
|
|
1940
|
+
// Get screen diff history
|
|
1941
|
+
ipcMain.handle('get-screen-diff-history', () => {
|
|
1942
|
+
return visualAwareness.getScreenDiffHistory();
|
|
1943
|
+
});
|
|
1944
|
+
|
|
1945
|
+
// ===== MULTI-AGENT SYSTEM IPC HANDLERS =====
|
|
1946
|
+
// Initialize agent system lazily
|
|
1947
|
+
let agentSystem = null;
|
|
1948
|
+
|
|
1949
|
+
function getAgentSystem() {
|
|
1950
|
+
if (!agentSystem) {
|
|
1951
|
+
agentSystem = createAgentSystem(aiService);
|
|
1952
|
+
}
|
|
1953
|
+
return agentSystem;
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
// Spawn a new agent session
|
|
1957
|
+
ipcMain.handle('agent-spawn', async (event, { task, options = {} }) => {
|
|
1958
|
+
try {
|
|
1959
|
+
const { orchestrator } = getAgentSystem();
|
|
1960
|
+
const sessionId = await orchestrator.startSession(task);
|
|
1961
|
+
|
|
1962
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1963
|
+
chatWindow.webContents.send('agent-event', {
|
|
1964
|
+
type: 'session-started',
|
|
1965
|
+
sessionId,
|
|
1966
|
+
task,
|
|
1967
|
+
timestamp: Date.now()
|
|
1968
|
+
});
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
return { success: true, sessionId };
|
|
1972
|
+
} catch (error) {
|
|
1973
|
+
console.error('[AGENT] Spawn failed:', error);
|
|
1974
|
+
return { success: false, error: error.message };
|
|
1975
|
+
}
|
|
1976
|
+
});
|
|
1977
|
+
|
|
1978
|
+
// Execute a task with the agent system
|
|
1979
|
+
ipcMain.handle('agent-run', async (event, { task, options = {} }) => {
|
|
1980
|
+
try {
|
|
1981
|
+
const { orchestrator } = getAgentSystem();
|
|
1982
|
+
|
|
1983
|
+
// Notify chat of execution start
|
|
1984
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1985
|
+
chatWindow.webContents.send('agent-event', {
|
|
1986
|
+
type: 'execution-started',
|
|
1987
|
+
task,
|
|
1988
|
+
timestamp: Date.now()
|
|
1989
|
+
});
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
const result = await orchestrator.orchestrate(task);
|
|
1993
|
+
|
|
1994
|
+
// Notify chat of completion
|
|
1995
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
1996
|
+
chatWindow.webContents.send('agent-event', {
|
|
1997
|
+
type: 'execution-complete',
|
|
1998
|
+
task,
|
|
1999
|
+
result,
|
|
2000
|
+
timestamp: Date.now()
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
return { success: true, result };
|
|
2005
|
+
} catch (error) {
|
|
2006
|
+
console.error('[AGENT] Run failed:', error);
|
|
2007
|
+
|
|
2008
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
2009
|
+
chatWindow.webContents.send('agent-event', {
|
|
2010
|
+
type: 'execution-error',
|
|
2011
|
+
task,
|
|
2012
|
+
error: error.message,
|
|
2013
|
+
timestamp: Date.now()
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
return { success: false, error: error.message };
|
|
2018
|
+
}
|
|
2019
|
+
});
|
|
2020
|
+
|
|
2021
|
+
// Research a topic using the researcher agent
|
|
2022
|
+
ipcMain.handle('agent-research', async (event, { query, options = {} }) => {
|
|
2023
|
+
try {
|
|
2024
|
+
const { orchestrator } = getAgentSystem();
|
|
2025
|
+
const result = await orchestrator.research(query);
|
|
2026
|
+
return { success: true, result };
|
|
2027
|
+
} catch (error) {
|
|
2028
|
+
console.error('[AGENT] Research failed:', error);
|
|
2029
|
+
return { success: false, error: error.message };
|
|
2030
|
+
}
|
|
2031
|
+
});
|
|
2032
|
+
|
|
2033
|
+
// Verify code/changes using the verifier agent
|
|
2034
|
+
ipcMain.handle('agent-verify', async (event, { target, options = {} }) => {
|
|
2035
|
+
try {
|
|
2036
|
+
const { orchestrator } = getAgentSystem();
|
|
2037
|
+
const result = await orchestrator.verify(target);
|
|
2038
|
+
return { success: true, result };
|
|
2039
|
+
} catch (error) {
|
|
2040
|
+
console.error('[AGENT] Verify failed:', error);
|
|
2041
|
+
return { success: false, error: error.message };
|
|
2042
|
+
}
|
|
2043
|
+
});
|
|
2044
|
+
|
|
2045
|
+
// Build code/features using the builder agent
|
|
2046
|
+
ipcMain.handle('agent-build', async (event, { specification, options = {} }) => {
|
|
2047
|
+
try {
|
|
2048
|
+
const { orchestrator } = getAgentSystem();
|
|
2049
|
+
const result = await orchestrator.build(specification);
|
|
2050
|
+
return { success: true, result };
|
|
2051
|
+
} catch (error) {
|
|
2052
|
+
console.error('[AGENT] Build failed:', error);
|
|
2053
|
+
return { success: false, error: error.message };
|
|
2054
|
+
}
|
|
2055
|
+
});
|
|
2056
|
+
|
|
2057
|
+
// Get agent system status
|
|
2058
|
+
ipcMain.handle('agent-status', async () => {
|
|
2059
|
+
try {
|
|
2060
|
+
const { stateManager, orchestrator } = getAgentSystem();
|
|
2061
|
+
const state = stateManager.getState();
|
|
2062
|
+
const currentSession = orchestrator.currentSession;
|
|
2063
|
+
|
|
2064
|
+
return {
|
|
2065
|
+
success: true,
|
|
2066
|
+
status: {
|
|
2067
|
+
initialized: !!agentSystem,
|
|
2068
|
+
currentSession,
|
|
2069
|
+
taskQueue: state.taskQueue.length,
|
|
2070
|
+
completedTasks: state.completedTasks.length,
|
|
2071
|
+
failedTasks: state.failedTasks.length,
|
|
2072
|
+
activeAgents: Object.keys(state.agents).filter(k => state.agents[k].currentTask).length,
|
|
2073
|
+
handoffCount: state.handoffs.length,
|
|
2074
|
+
sessions: state.sessions
|
|
2075
|
+
}
|
|
2076
|
+
};
|
|
2077
|
+
} catch (error) {
|
|
2078
|
+
console.error('[AGENT] Status failed:', error);
|
|
2079
|
+
return { success: false, error: error.message };
|
|
2080
|
+
}
|
|
2081
|
+
});
|
|
2082
|
+
|
|
2083
|
+
// Reset agent system state
|
|
2084
|
+
ipcMain.handle('agent-reset', async () => {
|
|
2085
|
+
try {
|
|
2086
|
+
const { stateManager } = getAgentSystem();
|
|
2087
|
+
stateManager.resetState();
|
|
2088
|
+
agentSystem = null; // Force re-initialization
|
|
2089
|
+
|
|
2090
|
+
return { success: true, message: 'Agent system reset successfully' };
|
|
2091
|
+
} catch (error) {
|
|
2092
|
+
console.error('[AGENT] Reset failed:', error);
|
|
2093
|
+
return { success: false, error: error.message };
|
|
2094
|
+
}
|
|
2095
|
+
});
|
|
2096
|
+
|
|
2097
|
+
// Get agent handoff history
|
|
2098
|
+
ipcMain.handle('agent-handoffs', async () => {
|
|
2099
|
+
try {
|
|
2100
|
+
const { stateManager } = getAgentSystem();
|
|
2101
|
+
const state = stateManager.getState();
|
|
2102
|
+
return { success: true, handoffs: state.handoffs };
|
|
2103
|
+
} catch (error) {
|
|
2104
|
+
console.error('[AGENT] Get handoffs failed:', error);
|
|
2105
|
+
return { success: false, error: error.message };
|
|
2106
|
+
}
|
|
2107
|
+
});
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
// ===== VISUAL CONTEXT MANAGEMENT (AI Awareness) =====
|
|
2111
|
+
let visualContextHistory = [];
|
|
2112
|
+
const MAX_VISUAL_CONTEXT_ITEMS = 10;
|
|
2113
|
+
|
|
2114
|
+
/**
|
|
2115
|
+
* Store visual context for AI processing
|
|
2116
|
+
*/
|
|
2117
|
+
function storeVisualContext(imageData) {
|
|
2118
|
+
visualContextHistory.push({
|
|
2119
|
+
...imageData,
|
|
2120
|
+
id: `vc-${Date.now()}`
|
|
2121
|
+
});
|
|
2122
|
+
|
|
2123
|
+
// Keep only recent items
|
|
2124
|
+
if (visualContextHistory.length > MAX_VISUAL_CONTEXT_ITEMS) {
|
|
2125
|
+
visualContextHistory.shift();
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
// Also add to AI service for vision capabilities
|
|
2129
|
+
aiService.addVisualContext(imageData);
|
|
2130
|
+
|
|
2131
|
+
// Notify chat window of visual context update
|
|
2132
|
+
if (chatWindow) {
|
|
2133
|
+
chatWindow.webContents.send('visual-context-update', {
|
|
2134
|
+
count: visualContextHistory.length,
|
|
2135
|
+
latest: imageData.timestamp
|
|
2136
|
+
});
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
/**
|
|
2141
|
+
* Initialize the application
|
|
2142
|
+
*/
|
|
2143
|
+
app.whenReady().then(() => {
|
|
2144
|
+
loadChatBoundsPrefs();
|
|
2145
|
+
createOverlayWindow();
|
|
2146
|
+
createChatWindow();
|
|
2147
|
+
createTray();
|
|
2148
|
+
registerShortcuts();
|
|
2149
|
+
setupIPC();
|
|
2150
|
+
|
|
2151
|
+
// Set up Copilot OAuth callback to notify chat on auth completion
|
|
2152
|
+
aiService.setOAuthCallback((result) => {
|
|
2153
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
2154
|
+
chatWindow.webContents.send('agent-response', {
|
|
2155
|
+
text: result.success ? result.message : `Authentication failed: ${result.message}`,
|
|
2156
|
+
type: result.success ? 'system' : 'error',
|
|
2157
|
+
timestamp: Date.now()
|
|
2158
|
+
});
|
|
2159
|
+
|
|
2160
|
+
// Also send auth status update
|
|
2161
|
+
chatWindow.webContents.send('auth-status', {
|
|
2162
|
+
provider: 'copilot',
|
|
2163
|
+
status: result.success ? 'connected' : 'error'
|
|
2164
|
+
});
|
|
2165
|
+
}
|
|
2166
|
+
});
|
|
2167
|
+
|
|
2168
|
+
// Try to load saved Copilot token
|
|
2169
|
+
aiService.loadCopilotToken();
|
|
2170
|
+
|
|
2171
|
+
// Send initial auth status after a short delay (wait for chat window to be ready)
|
|
2172
|
+
setTimeout(() => {
|
|
2173
|
+
if (chatWindow && !chatWindow.isDestroyed()) {
|
|
2174
|
+
const status = aiService.getStatus();
|
|
2175
|
+
const tokenPath = require('path').join(app.getPath('appData'), 'copilot-agent', 'copilot-token.json');
|
|
2176
|
+
const hasCopilotToken = require('fs').existsSync(tokenPath);
|
|
2177
|
+
|
|
2178
|
+
chatWindow.webContents.send('auth-status', {
|
|
2179
|
+
provider: status.provider,
|
|
2180
|
+
status: hasCopilotToken ? 'connected' : 'disconnected'
|
|
2181
|
+
});
|
|
2182
|
+
}
|
|
2183
|
+
}, 1000);
|
|
2184
|
+
|
|
2185
|
+
app.on('activate', () => {
|
|
2186
|
+
if (BrowserWindow.getAllWindows().length === 0) {
|
|
2187
|
+
createOverlayWindow();
|
|
2188
|
+
createChatWindow();
|
|
2189
|
+
}
|
|
2190
|
+
});
|
|
2191
|
+
});
|
|
2192
|
+
|
|
2193
|
+
// Quit when all windows are closed (except on macOS)
|
|
2194
|
+
app.on('window-all-closed', () => {
|
|
2195
|
+
if (process.platform !== 'darwin') {
|
|
2196
|
+
app.quit();
|
|
2197
|
+
}
|
|
2198
|
+
});
|
|
2199
|
+
|
|
2200
|
+
// Clean up shortcuts on quit
|
|
2201
|
+
app.on('will-quit', () => {
|
|
2202
|
+
globalShortcut.unregisterAll();
|
|
2203
|
+
});
|
|
2204
|
+
|
|
2205
|
+
// Prevent app from quitting when closing chat window
|
|
2206
|
+
app.on('before-quit', () => {
|
|
2207
|
+
app.isQuitting = true;
|
|
2208
|
+
});
|