copilot-liku-cli 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/ARCHITECTURE.md +411 -0
  2. package/CONFIGURATION.md +302 -0
  3. package/CONTRIBUTING.md +225 -0
  4. package/ELECTRON_README.md +121 -0
  5. package/INSTALLATION.md +350 -0
  6. package/LICENSE.md +1 -0
  7. package/PROJECT_STATUS.md +229 -0
  8. package/QUICKSTART.md +255 -0
  9. package/README.md +167 -0
  10. package/TESTING.md +274 -0
  11. package/package.json +61 -0
  12. package/scripts/start.js +30 -0
  13. package/src/assets/tray-icon.png +0 -0
  14. package/src/cli/commands/agent.js +327 -0
  15. package/src/cli/commands/click.js +108 -0
  16. package/src/cli/commands/drag.js +85 -0
  17. package/src/cli/commands/find.js +109 -0
  18. package/src/cli/commands/keys.js +132 -0
  19. package/src/cli/commands/mouse.js +79 -0
  20. package/src/cli/commands/repl.js +290 -0
  21. package/src/cli/commands/screenshot.js +72 -0
  22. package/src/cli/commands/scroll.js +74 -0
  23. package/src/cli/commands/start.js +67 -0
  24. package/src/cli/commands/type.js +57 -0
  25. package/src/cli/commands/wait.js +84 -0
  26. package/src/cli/commands/window.js +104 -0
  27. package/src/cli/liku.js +249 -0
  28. package/src/cli/util/output.js +174 -0
  29. package/src/main/agents/base-agent.js +410 -0
  30. package/src/main/agents/builder.js +484 -0
  31. package/src/main/agents/index.js +62 -0
  32. package/src/main/agents/orchestrator.js +362 -0
  33. package/src/main/agents/researcher.js +511 -0
  34. package/src/main/agents/state-manager.js +344 -0
  35. package/src/main/agents/supervisor.js +365 -0
  36. package/src/main/agents/verifier.js +452 -0
  37. package/src/main/ai-service.js +1633 -0
  38. package/src/main/index.js +2208 -0
  39. package/src/main/inspect-service.js +467 -0
  40. package/src/main/system-automation.js +1186 -0
  41. package/src/main/ui-automation/config.js +76 -0
  42. package/src/main/ui-automation/core/helpers.js +41 -0
  43. package/src/main/ui-automation/core/index.js +15 -0
  44. package/src/main/ui-automation/core/powershell.js +82 -0
  45. package/src/main/ui-automation/elements/finder.js +274 -0
  46. package/src/main/ui-automation/elements/index.js +14 -0
  47. package/src/main/ui-automation/elements/wait.js +66 -0
  48. package/src/main/ui-automation/index.js +164 -0
  49. package/src/main/ui-automation/interactions/element-click.js +211 -0
  50. package/src/main/ui-automation/interactions/high-level.js +230 -0
  51. package/src/main/ui-automation/interactions/index.js +47 -0
  52. package/src/main/ui-automation/keyboard/index.js +15 -0
  53. package/src/main/ui-automation/keyboard/input.js +179 -0
  54. package/src/main/ui-automation/mouse/click.js +186 -0
  55. package/src/main/ui-automation/mouse/drag.js +88 -0
  56. package/src/main/ui-automation/mouse/index.js +30 -0
  57. package/src/main/ui-automation/mouse/movement.js +51 -0
  58. package/src/main/ui-automation/mouse/scroll.js +116 -0
  59. package/src/main/ui-automation/screenshot.js +183 -0
  60. package/src/main/ui-automation/window/index.js +23 -0
  61. package/src/main/ui-automation/window/manager.js +305 -0
  62. package/src/main/utils/time.js +62 -0
  63. package/src/main/visual-awareness.js +597 -0
  64. package/src/renderer/chat/chat.js +671 -0
  65. package/src/renderer/chat/index.html +725 -0
  66. package/src/renderer/chat/preload.js +112 -0
  67. package/src/renderer/overlay/index.html +648 -0
  68. package/src/renderer/overlay/overlay.js +782 -0
  69. package/src/renderer/overlay/preload.js +90 -0
  70. package/src/shared/grid-math.js +82 -0
  71. package/src/shared/inspect-types.js +230 -0
@@ -0,0 +1,2208 @@
1
+ // Ensure Electron runs in app mode even if a dev shell has ELECTRON_RUN_AS_NODE set
2
+ if (process.env.ELECTRON_RUN_AS_NODE) {
3
+ console.warn('ELECTRON_RUN_AS_NODE was set; clearing so the app can start normally.');
4
+ delete process.env.ELECTRON_RUN_AS_NODE;
5
+ }
6
+
7
+ const {
8
+ app,
9
+ BrowserWindow,
10
+ Tray,
11
+ Menu,
12
+ globalShortcut,
13
+ ipcMain,
14
+ screen,
15
+ nativeImage,
16
+ desktopCapturer
17
+ } = require('electron');
18
+ const path = require('path');
19
+ const fs = require('fs');
20
+ const os = require('os');
21
+
22
+ // AI Service for handling chat responses
23
+ const aiService = require('./ai-service.js');
24
+
25
+ // Visual awareness for advanced screen analysis
26
+ const visualAwareness = require('./visual-awareness.js');
27
+
28
+ // Multi-agent system for advanced AI orchestration
29
+ const { createAgentSystem } = require('./agents/index.js');
30
+
31
+ // Inspect service for overlay region detection and targeting
32
+ const inspectService = require('./inspect-service.js');
33
+
34
+
35
+ // Ensure caches land in a writable location to avoid Windows permission issues
36
+ const cacheRoot = path.join(os.tmpdir(), 'copilot-liku-electron-cache');
37
+ const mediaCache = path.join(cacheRoot, 'media');
38
+ const userDataPath = path.join(cacheRoot, 'user-data');
39
+
40
+ try {
41
+ fs.mkdirSync(cacheRoot, { recursive: true });
42
+ fs.mkdirSync(mediaCache, { recursive: true });
43
+ fs.mkdirSync(userDataPath, { recursive: true });
44
+
45
+ // Force Electron to use temp-backed storage to avoid permission issues on locked-down drives
46
+ app.setPath('userData', userDataPath);
47
+ app.setPath('cache', cacheRoot);
48
+
49
+ app.commandLine.appendSwitch('disk-cache-dir', cacheRoot);
50
+ app.commandLine.appendSwitch('media-cache-dir', mediaCache);
51
+ app.commandLine.appendSwitch('disable-gpu-shader-disk-cache');
52
+ } catch (error) {
53
+ console.warn('Unable to create cache directories; continuing with defaults.', error);
54
+ }
55
+
56
+ // Keep references to windows to prevent garbage collection
57
+ let overlayWindow = null;
58
+ let chatWindow = null;
59
+ let tray = null;
60
+
61
+ // State management
62
+ let overlayMode = 'selection'; // start in selection so the grid is visible immediately
63
+ let isChatVisible = false;
64
+
65
+ /**
66
+ * Create the transparent overlay window that floats above all other windows
67
+ */
68
+ function createOverlayWindow() {
69
+ const { width, height } = screen.getPrimaryDisplay().bounds;
70
+
71
+ overlayWindow = new BrowserWindow({
72
+ width,
73
+ height,
74
+ frame: false,
75
+ transparent: true,
76
+ alwaysOnTop: true,
77
+ skipTaskbar: true,
78
+ resizable: false,
79
+ movable: false,
80
+ minimizable: false,
81
+ maximizable: false,
82
+ closable: false,
83
+ focusable: true,
84
+ hasShadow: false,
85
+ webPreferences: {
86
+ nodeIntegration: false,
87
+ contextIsolation: true,
88
+ preload: path.join(__dirname, '../renderer/overlay/preload.js')
89
+ }
90
+ });
91
+
92
+ // Set highest level for macOS to float above fullscreen apps
93
+ if (process.platform === 'darwin') {
94
+ overlayWindow.setAlwaysOnTop(true, 'screen-saver');
95
+ overlayWindow.setFullScreen(true);
96
+ } else {
97
+ // On Windows: Use maximize instead of fullscreen to avoid interfering with other windows
98
+ overlayWindow.setAlwaysOnTop(true, 'screen-saver');
99
+ overlayWindow.maximize();
100
+ overlayWindow.setPosition(0, 0);
101
+ }
102
+
103
+ // Start in click-through mode
104
+ overlayWindow.setIgnoreMouseEvents(true, { forward: true });
105
+
106
+ overlayWindow.loadFile(path.join(__dirname, '../renderer/overlay/index.html'));
107
+
108
+ // Once the overlay loads, ensure it is visible and interactive
109
+ overlayWindow.webContents.on('did-finish-load', () => {
110
+ overlayWindow.show();
111
+ setOverlayMode('selection');
112
+ });
113
+
114
+ // Pipe renderer console to main for debugging without DevTools
115
+ overlayWindow.webContents.on('console-message', (event) => {
116
+ const { level, message, line, sourceId } = event;
117
+ console.log(`[overlay console] (${level}) ${sourceId}:${line} - ${message}`);
118
+ });
119
+
120
+ // Prevent overlay from appearing in Dock/Taskbar
121
+ if (process.platform === 'darwin') {
122
+ app.dock.hide();
123
+ }
124
+
125
+ overlayWindow.on('closed', () => {
126
+ overlayWindow = null;
127
+ });
128
+ }
129
+
130
+ // Chat window position preferences (persisted)
131
+ let chatBoundsPrefs = null;
132
+
133
+ function loadChatBoundsPrefs() {
134
+ try {
135
+ const prefsPath = path.join(userDataPath, 'chat-bounds.json');
136
+ if (fs.existsSync(prefsPath)) {
137
+ chatBoundsPrefs = JSON.parse(fs.readFileSync(prefsPath, 'utf8'));
138
+ console.log('Loaded chat bounds preferences:', chatBoundsPrefs);
139
+ }
140
+ } catch (e) {
141
+ console.warn('Could not load chat bounds preferences:', e);
142
+ }
143
+ }
144
+
145
+ function saveChatBoundsPrefs(bounds) {
146
+ try {
147
+ const prefsPath = path.join(userDataPath, 'chat-bounds.json');
148
+ fs.writeFileSync(prefsPath, JSON.stringify(bounds));
149
+ chatBoundsPrefs = bounds;
150
+ } catch (e) {
151
+ console.warn('Could not save chat bounds preferences:', e);
152
+ }
153
+ }
154
+
155
+ /**
156
+ * Create the chat window positioned at screen edge (bottom-right)
157
+ * FRESH APPROACH: Create window with absolute minimal config, position AFTER creation
158
+ */
159
+ function createChatWindow() {
160
+ // Destroy existing window if any
161
+ if (chatWindow) {
162
+ chatWindow.destroy();
163
+ chatWindow = null;
164
+ }
165
+
166
+ const display = screen.getPrimaryDisplay();
167
+ const { width: screenWidth, height: screenHeight } = display.workAreaSize;
168
+
169
+ // HARDCODED small window - bottom right
170
+ const W = 380;
171
+ const H = 500;
172
+ const X = screenWidth - W - 20;
173
+ const Y = screenHeight - H - 20;
174
+
175
+ console.log(`[CHAT] Creating at ${X},${Y} size ${W}x${H}`);
176
+
177
+ chatWindow = new BrowserWindow({
178
+ width: W,
179
+ height: H,
180
+ x: X,
181
+ y: Y,
182
+ minWidth: 300,
183
+ minHeight: 400,
184
+ maxWidth: 600,
185
+ maxHeight: 800,
186
+ frame: false,
187
+ transparent: false,
188
+ resizable: true,
189
+ minimizable: true,
190
+ maximizable: false,
191
+ fullscreenable: false,
192
+ alwaysOnTop: false,
193
+ skipTaskbar: false,
194
+ show: false,
195
+ backgroundColor: '#1e1e1e',
196
+ webPreferences: {
197
+ nodeIntegration: false,
198
+ contextIsolation: true,
199
+ preload: path.join(__dirname, '../renderer/chat/preload.js')
200
+ }
201
+ });
202
+
203
+ // Immediately set bounds again
204
+ chatWindow.setBounds({ x: X, y: Y, width: W, height: H });
205
+
206
+ chatWindow.loadFile(path.join(__dirname, '../renderer/chat/index.html'));
207
+
208
+ const persistBounds = () => {
209
+ if (!chatWindow) return;
210
+ saveChatBoundsPrefs(chatWindow.getBounds());
211
+ };
212
+
213
+ chatWindow.webContents.on('did-finish-load', () => {
214
+ // Force bounds one more time after load
215
+ chatWindow.setBounds({ x: X, y: Y, width: W, height: H });
216
+ console.log(`[CHAT] Loaded. Bounds: ${JSON.stringify(chatWindow.getBounds())}`);
217
+ });
218
+
219
+ chatWindow.on('resize', persistBounds);
220
+ chatWindow.on('move', persistBounds);
221
+
222
+ chatWindow.on('close', (event) => {
223
+ if (!app.isQuitting) {
224
+ event.preventDefault();
225
+ chatWindow.hide();
226
+ isChatVisible = false;
227
+ }
228
+ });
229
+
230
+ chatWindow.on('closed', () => {
231
+ chatWindow = null;
232
+ });
233
+ }
234
+
235
+ /**
236
+ * Toggle chat - recreate window fresh each time to avoid fullscreen bug
237
+ */
238
+ function toggleChat() {
239
+ if (chatWindow && chatWindow.isVisible()) {
240
+ chatWindow.hide();
241
+ isChatVisible = false;
242
+ return;
243
+ }
244
+
245
+ // RECREATE window fresh each time
246
+ createChatWindow();
247
+
248
+ // Show after a brief delay to ensure bounds are set
249
+ setTimeout(() => {
250
+ if (chatWindow) {
251
+ const display = screen.getPrimaryDisplay();
252
+ const { width: screenWidth, height: screenHeight } = display.workAreaSize;
253
+ const W = 380, H = 500;
254
+ const X = screenWidth - W - 20;
255
+ const Y = screenHeight - H - 20;
256
+
257
+ // AGGRESSIVE: Multiple setters to override any system defaults
258
+ chatWindow.unmaximize();
259
+ chatWindow.setFullScreen(false);
260
+ chatWindow.setSize(W, H);
261
+ chatWindow.setPosition(X, Y);
262
+ chatWindow.setBounds({ x: X, y: Y, width: W, height: H });
263
+
264
+ chatWindow.show();
265
+ chatWindow.focus();
266
+
267
+ // AFTER show: force bounds again
268
+ chatWindow.setSize(W, H);
269
+ chatWindow.setPosition(X, Y);
270
+
271
+ isChatVisible = true;
272
+ console.log(`[CHAT] Shown. Final bounds: ${JSON.stringify(chatWindow.getBounds())}`);
273
+
274
+ // Validate bounds after 200ms and correct if needed
275
+ setTimeout(() => {
276
+ if (chatWindow) {
277
+ const bounds = chatWindow.getBounds();
278
+ if (bounds.width !== W || bounds.height !== H) {
279
+ console.log(`[CHAT] CORRECTING: Bounds were ${JSON.stringify(bounds)}, forcing to ${W}x${H}@${X},${Y}`);
280
+ chatWindow.setSize(W, H);
281
+ chatWindow.setPosition(X, Y);
282
+ }
283
+ }
284
+ }, 200);
285
+ }
286
+ }, 100);
287
+ }
288
+
289
+ /**
290
+ * Create system tray icon with menu
291
+ */
292
+ function loadTrayIcon() {
293
+ const candidates = [
294
+ path.join(__dirname, '../assets/tray-icon.png'),
295
+ path.join(app.getAppPath(), 'src/assets/tray-icon.png'),
296
+ path.join(process.resourcesPath, 'assets', 'tray-icon.png'),
297
+ path.join(process.resourcesPath, 'tray-icon.png')
298
+ ];
299
+
300
+ for (const candidate of candidates) {
301
+ try {
302
+ if (!fs.existsSync(candidate)) {
303
+ continue;
304
+ }
305
+
306
+ const image = nativeImage.createFromPath(candidate);
307
+
308
+ if (!image.isEmpty()) {
309
+ return { image, source: candidate };
310
+ }
311
+
312
+ console.warn(`Tray icon candidate was empty: ${candidate}`);
313
+ } catch (error) {
314
+ console.warn(`Tray icon candidate failed (${candidate}):`, error);
315
+ }
316
+ }
317
+
318
+ const fallbackBase64 = 'iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAASbSURBVFhHxZf7U1R1FMDv34E8hGW5LBcMAiQiQkMgogVqYW1FA1EDRF6Rg5oMp0hn0CYQB4KBFAlGGoQZgtmaxmqRQWCkSIpVA7aUjZeJkI8RrTnN+a57Y5e7j/xlzy935nvvPZ/z/n6/nCDEbAkK2gpBz8WCEJIAQkgiCKFKEEJTQAhXgRCeDkKEBhSRGaCIzARF1G5QRO0DRXQO8NH5wMcUAB9TAvzWUuBjy4CPPQx8XDnwcRUgT6gEecIxkCdWgTzpJMiTqkGmPAUyZR3IUhpBltIEHIMHb8PAkAQMfP41DAxVYmBYKgrhKhQ2q1GI0GBA5E4MeDETFVHZqHhpHyqic9H/5Xz0jylE/y0l6P/Ke8jHliG/7QjyceXIxwP6JVSi36vH0S+xCuVJH6P89Wr0Vdaib3Id+qY0oiy1GWVvnEWOPHcVXPZmK3IUdlfBZWntZEAiuAruk96BHBWcq+A+6k4ygFW7S+A+27uRY632jPDUvAbcebAV1cXnngnu81YPGcD63Gn43sMtOPKTAaVk7PosftR0yWm4t6YPORoyzsA3Jb6PLV2DFkDj/DKOXL2J0zN3LNb7f7iFL2S1O4R77/gKOZpwzsD1U7MioLFjEMNUn1iEXVDVYXm9Dlfur7Jv6Blf0GMX7p3xDRlA49U2nMJ+vneEKTXO30V10Rm7BUeeD/88x74fu3EbFTs6bcI37voWOTbb7cDzK9qYskerT9bBkwvbMONIFyqLvrDIecD2dpxZvM/+O9U5YRO+8W0dGUAbizScql0/ZfLm9Of9IjxUfRr7R397mhCTjN1YwPgD3WLONfCdyfDHf6NiV68k3CtzADna1WzBNyV9IAKCkk+Ing+Pz6xB/ydTxmUM0HSIOZ/64y+2nnTokiTcK+sycrSlSsGpz9VFzUyBfnpehKeXdlhhLSXnxICY895BI1s7+tkvknCv3SPI0X4uBacJV1GrZQr6dHqx4MrrTaG1JWtzXnX+GltruzgjCffMHiUDSkAKTuN1z1GTt2PXZsVqz4ZeK6SlHKwfFQuu7eIttvZh26+ScM89Y8g9Pcmsg1PBhaXVMAXUAeZWE9KbcWbhnhXWJFRwEblfiwU38bupBjJPjkvCPfeOI8eOURJwc86NCytMSd5xrdhqqrI+XLz7cB28sHZUhG/er8NHj/9h74LyhiThnu9MkAHsDCcJp7A3dI4yJYtLDzAko1UcryGZnVjZ8iPLeeW5cQvPqeB0V/9k/+nGl2zCPXKuI8cOkDbgFHYhrQH1BpOy3gGDw9lOcMo5ycqDJxhWdMUm3CN3kgyoAFtw83iNz7/AQkxC4zWuWCsJp7CbPScpbZ60C/fIMyBHR2d7cPN4VR3SiuOVZHjiNp7RTrNW+7RnGof0S2LOyXNn4B77b5IBx8AR3Bz24KwuvPC95Qi2Fsq5o7Cb4e75RuRMlwbH8LU5D87WoqZyCGu6JvHLy3NY021grRZVesVuwVnD3Q/MIcduLP8Dbl1wtiacM3D3gkUyoBpcBd9QdAc5uqu5Cr6heJkMqANXwd1K7iHHbqkugru9+5AMaAJXwd1KV/Ff/Hw4CMaLXiMAAAAASUVORK5CYII=';
319
+ const fallbackImage = nativeImage.createFromDataURL(`data:image/png;base64,${fallbackBase64}`);
320
+
321
+ return { image: fallbackImage, source: 'embedded-fallback' };
322
+ }
323
+
324
+ function createTray() {
325
+ const { image: trayIcon, source } = loadTrayIcon();
326
+
327
+ try {
328
+ tray = new Tray(trayIcon);
329
+ } catch (error) {
330
+ console.error('Failed to initialize tray icon:', error);
331
+ tray = new Tray(nativeImage.createEmpty());
332
+ }
333
+
334
+ if (source === 'embedded-fallback') {
335
+ console.warn('Using embedded fallback tray icon because no valid asset was found.');
336
+ } else {
337
+ console.log(`Tray icon loaded from: ${source}`);
338
+ }
339
+
340
+ const contextMenu = Menu.buildFromTemplate([
341
+ {
342
+ label: 'Open Chat',
343
+ click: () => toggleChat()
344
+ },
345
+ {
346
+ label: 'Toggle Overlay',
347
+ click: () => toggleOverlay()
348
+ },
349
+ { type: 'separator' },
350
+ {
351
+ label: 'Reset Window Positions',
352
+ click: () => {
353
+ // Clear saved preferences and reset both windows
354
+ chatBoundsPrefs = null;
355
+ try {
356
+ const prefsPath = path.join(userDataPath, 'chat-bounds.json');
357
+ if (fs.existsSync(prefsPath)) fs.unlinkSync(prefsPath);
358
+ } catch (e) {}
359
+ ensureChatBounds(true);
360
+ if (chatWindow && chatWindow.isVisible()) {
361
+ chatWindow.show();
362
+ chatWindow.focus();
363
+ }
364
+ }
365
+ },
366
+ { type: 'separator' },
367
+ {
368
+ label: 'Quit',
369
+ click: () => {
370
+ app.isQuitting = true;
371
+ app.quit();
372
+ }
373
+ }
374
+ ]);
375
+
376
+ tray.setToolTip('Copilot Agent Overlay');
377
+ tray.setContextMenu(contextMenu);
378
+
379
+ // On macOS, clicking tray icon shows chat
380
+ tray.on('click', () => {
381
+ toggleChat();
382
+ });
383
+ }
384
+
385
+ /**
386
+ * Ensure chat window has valid bounds (not off-screen, not fullscreen)
387
+ */
388
+ function ensureChatBounds(force = false) {
389
+ if (!chatWindow) return;
390
+
391
+ // Always ensure not fullscreen
392
+ if (chatWindow.isFullScreen()) {
393
+ chatWindow.setFullScreen(false);
394
+ }
395
+
396
+ const { width, height } = screen.getPrimaryDisplay().workAreaSize;
397
+ const bounds = chatWindow.getBounds();
398
+
399
+ // Check if off-screen
400
+ const isOffScreen = bounds.x < -bounds.width ||
401
+ bounds.x > width ||
402
+ bounds.y < -bounds.height ||
403
+ bounds.y > height;
404
+
405
+ // Check if too large for screen
406
+ const isTooLarge = bounds.width > width || bounds.height > height;
407
+
408
+ if (force || isOffScreen || isTooLarge) {
409
+ if (chatWindow.isMaximized()) {
410
+ chatWindow.unmaximize();
411
+ }
412
+
413
+ // Use saved preferences or calculate default bottom-right position
414
+ const defaultWidth = chatBoundsPrefs?.width || 380;
415
+ const defaultHeight = chatBoundsPrefs?.height || 520;
416
+ const margin = 20;
417
+
418
+ chatWindow.setBounds({
419
+ width: Math.min(defaultWidth, width - margin * 2),
420
+ height: Math.min(defaultHeight, height - margin * 2),
421
+ x: chatBoundsPrefs?.x ?? Math.max(0, width - defaultWidth - margin),
422
+ y: chatBoundsPrefs?.y ?? Math.max(0, height - defaultHeight - margin)
423
+ });
424
+ }
425
+ }
426
+
427
+ /**
428
+ * Toggle overlay visibility
429
+ */
430
+ function toggleOverlay() {
431
+ if (!overlayWindow) return;
432
+
433
+ if (overlayWindow.isVisible()) {
434
+ overlayWindow.hide();
435
+ setOverlayMode('passive');
436
+ } else {
437
+ overlayWindow.show();
438
+ setOverlayMode('selection');
439
+ }
440
+ }
441
+
442
+ /**
443
+ * Set overlay mode (passive or selection)
444
+ *
445
+ * CRITICAL: We ALWAYS use setIgnoreMouseEvents(true, { forward: true }) so that
446
+ * clicks pass through to background applications. The overlay dots use CSS
447
+ * pointer-events: auto to still receive clicks when hovered. This is the
448
+ * correct pattern for transparent overlays with clickable elements.
449
+ */
450
+ function setOverlayMode(mode) {
451
+ overlayMode = mode;
452
+
453
+ if (!overlayWindow) return;
454
+
455
+ // ALWAYS forward mouse events to apps beneath the overlay.
456
+ // Dots with pointer-events: auto in CSS will still receive clicks.
457
+ overlayWindow.setIgnoreMouseEvents(true, { forward: true });
458
+
459
+ if (mode === 'passive') {
460
+ overlayWindow.setFocusable(false);
461
+ unregisterOverlayShortcuts();
462
+ } else if (mode === 'selection') {
463
+ // In selection mode, allow the window to be focusable for keyboard events
464
+ if (typeof overlayWindow.setFocusable === 'function') {
465
+ overlayWindow.setFocusable(true);
466
+ }
467
+ registerOverlayShortcuts();
468
+ }
469
+
470
+ // Notify overlay renderer of mode change
471
+ overlayWindow.webContents.send('mode-changed', mode);
472
+ console.log(`Overlay mode set to ${mode} (click-through enabled, dots are clickable via CSS)`);
473
+ }
474
+
475
+ /**
476
+ * Register overlay-specific shortcuts when in selection mode
477
+ * These use globalShortcut because the overlay has setIgnoreMouseEvents(true)
478
+ * which means keyboard events go to background apps, not the overlay window
479
+ */
480
+ function registerOverlayShortcuts() {
481
+ console.log('[SHORTCUTS] Registering overlay shortcuts (Ctrl+Alt+F/G/+/-/X/I)');
482
+
483
+ // Ctrl+Alt+F to toggle fine grid
484
+ globalShortcut.register('CommandOrControl+Alt+F', () => {
485
+ if (overlayWindow && overlayMode === 'selection') {
486
+ console.log('[SHORTCUTS] Ctrl+Alt+F pressed - toggle fine grid');
487
+ console.log('[SHORTCUTS] overlayWindow destroyed?', overlayWindow.isDestroyed());
488
+ console.log('[SHORTCUTS] Sending overlay-command to webContents');
489
+ overlayWindow.webContents.send('overlay-command', { action: 'toggle-fine' });
490
+ console.log('[SHORTCUTS] Sent overlay-command');
491
+ } else {
492
+ console.log('[SHORTCUTS] Ctrl+Alt+F pressed but not in selection mode or no overlay');
493
+ }
494
+ });
495
+
496
+ // Ctrl+Alt+G to show all grids
497
+ globalShortcut.register('CommandOrControl+Alt+G', () => {
498
+ if (overlayWindow && overlayMode === 'selection') {
499
+ console.log('[SHORTCUTS] Ctrl+Alt+G pressed - show all grids');
500
+ overlayWindow.webContents.send('overlay-command', { action: 'show-all' });
501
+ }
502
+ });
503
+
504
+ // Ctrl+Alt+= to zoom in
505
+ globalShortcut.register('CommandOrControl+Alt+=', () => {
506
+ if (overlayWindow && overlayMode === 'selection') {
507
+ console.log('[SHORTCUTS] Ctrl+Alt+= pressed - zoom in');
508
+ overlayWindow.webContents.send('overlay-command', { action: 'zoom-in' });
509
+ }
510
+ });
511
+
512
+ // Ctrl+Alt+- to zoom out
513
+ globalShortcut.register('CommandOrControl+Alt+-', () => {
514
+ if (overlayWindow && overlayMode === 'selection') {
515
+ console.log('[SHORTCUTS] Ctrl+Alt+- pressed - zoom out');
516
+ overlayWindow.webContents.send('overlay-command', { action: 'zoom-out' });
517
+ }
518
+ });
519
+
520
+ // Ctrl+Alt+X to cancel selection
521
+ globalShortcut.register('CommandOrControl+Alt+X', () => {
522
+ if (overlayWindow && overlayMode === 'selection') {
523
+ console.log('[SHORTCUTS] Ctrl+Alt+X pressed - cancel');
524
+ overlayWindow.webContents.send('overlay-command', { action: 'cancel' });
525
+ }
526
+ });
527
+
528
+ // Ctrl+Alt+I to toggle inspect mode
529
+ globalShortcut.register('CommandOrControl+Alt+I', () => {
530
+ if (overlayWindow && overlayMode === 'selection') {
531
+ console.log('[SHORTCUTS] Ctrl+Alt+I pressed - toggle inspect mode');
532
+ // Toggle inspect mode via IPC
533
+ const newState = !inspectService.isInspectModeActive();
534
+ inspectService.setInspectMode(newState);
535
+
536
+ // Notify overlay
537
+ overlayWindow.webContents.send('inspect-mode-changed', newState);
538
+ overlayWindow.webContents.send('overlay-command', { action: 'toggle-inspect' });
539
+
540
+ // If enabled, trigger region detection
541
+ if (newState) {
542
+ // Use async detection with error handling
543
+ inspectService.detectRegions().then(results => {
544
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
545
+ overlayWindow.webContents.send('inspect-regions-update', results.regions);
546
+ }
547
+ }).catch(err => {
548
+ console.error('[SHORTCUTS] Inspect region detection failed:', err);
549
+ });
550
+ }
551
+ }
552
+ });
553
+ }
554
+
555
+ /**
556
+ * Unregister overlay-specific shortcuts when leaving selection mode
557
+ */
558
+ function unregisterOverlayShortcuts() {
559
+ console.log('[SHORTCUTS] Unregistering overlay shortcuts');
560
+ const keys = [
561
+ 'CommandOrControl+Alt+F',
562
+ 'CommandOrControl+Alt+G',
563
+ 'CommandOrControl+Alt+=',
564
+ 'CommandOrControl+Alt+-',
565
+ 'CommandOrControl+Alt+X',
566
+ 'CommandOrControl+Alt+I'
567
+ ];
568
+ keys.forEach(key => {
569
+ try {
570
+ globalShortcut.unregister(key);
571
+ } catch (e) {
572
+ // Ignore errors if shortcut wasn't registered
573
+ }
574
+ });
575
+ }
576
+
577
+ /**
578
+ * Register global shortcuts
579
+ */
580
+ function registerShortcuts() {
581
+ // Ctrl+Alt+Space to toggle chat
582
+ globalShortcut.register('CommandOrControl+Alt+Space', () => {
583
+ toggleChat();
584
+ });
585
+
586
+ // Ctrl+Shift+O to toggle overlay
587
+ globalShortcut.register('CommandOrControl+Shift+O', () => {
588
+ toggleOverlay();
589
+ });
590
+ }
591
+
592
+ /**
593
+ * Set up IPC handlers
594
+ */
595
+ function setupIPC() {
596
+ // Handle dot selection from overlay
597
+ ipcMain.on('dot-selected', (event, data) => {
598
+ console.log('Dot selected:', data);
599
+
600
+ // Forward to chat window
601
+ if (chatWindow) {
602
+ chatWindow.webContents.send('dot-selected', data);
603
+ }
604
+
605
+ // Switch back to passive mode after selection (unless cancelled)
606
+ if (!data.cancelled) {
607
+ setOverlayMode('passive');
608
+ }
609
+ });
610
+
611
+ // Handle mode change requests from chat
612
+ ipcMain.on('set-mode', (event, mode) => {
613
+ setOverlayMode(mode);
614
+ });
615
+
616
+ // Agentic mode flag (when true, actions execute automatically)
617
+ let agenticMode = false;
618
+ let pendingActions = null;
619
+
620
+ // Handle chat messages
621
+ ipcMain.on('chat-message', async (event, message) => {
622
+ console.log('Chat message:', message);
623
+
624
+ // Check for slash commands first
625
+ if (message.startsWith('/')) {
626
+ // Handle agentic mode toggle
627
+ if (message === '/agentic' || message === '/agent') {
628
+ agenticMode = !agenticMode;
629
+ if (chatWindow) {
630
+ chatWindow.webContents.send('agent-response', {
631
+ text: `Agentic mode ${agenticMode ? 'ENABLED' : 'DISABLED'}. ${agenticMode ? 'Actions will execute automatically.' : 'Actions will require confirmation.'}`,
632
+ type: 'system',
633
+ timestamp: Date.now()
634
+ });
635
+ }
636
+ return;
637
+ }
638
+
639
+ // ===== MULTI-AGENT SYSTEM COMMANDS =====
640
+ // /orchestrate - Run full orchestration on a task
641
+ if (message.startsWith('/orchestrate ')) {
642
+ const task = message.slice('/orchestrate '.length).trim();
643
+ if (chatWindow) {
644
+ chatWindow.webContents.send('agent-response', {
645
+ text: `🎭 Starting multi-agent orchestration for: "${task}"`,
646
+ type: 'system',
647
+ timestamp: Date.now()
648
+ });
649
+ chatWindow.webContents.send('agent-typing', { isTyping: true });
650
+ }
651
+
652
+ try {
653
+ const { orchestrator } = getAgentSystem();
654
+ const result = await orchestrator.orchestrate(task);
655
+
656
+ if (chatWindow) {
657
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
658
+ chatWindow.webContents.send('agent-response', {
659
+ text: `🎭 Orchestration complete:\n\n${JSON.stringify(result, null, 2)}`,
660
+ type: result.status === 'success' ? 'message' : 'error',
661
+ timestamp: Date.now()
662
+ });
663
+ }
664
+ } catch (error) {
665
+ if (chatWindow) {
666
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
667
+ chatWindow.webContents.send('agent-response', {
668
+ text: `❌ Orchestration failed: ${error.message}`,
669
+ type: 'error',
670
+ timestamp: Date.now()
671
+ });
672
+ }
673
+ }
674
+ return;
675
+ }
676
+
677
+ // /research - Use researcher agent
678
+ if (message.startsWith('/research ')) {
679
+ const query = message.slice('/research '.length).trim();
680
+ if (chatWindow) {
681
+ chatWindow.webContents.send('agent-response', {
682
+ text: `🔍 Researching: "${query}"`,
683
+ type: 'system',
684
+ timestamp: Date.now()
685
+ });
686
+ chatWindow.webContents.send('agent-typing', { isTyping: true });
687
+ }
688
+
689
+ try {
690
+ const { orchestrator } = getAgentSystem();
691
+ const result = await orchestrator.research(query);
692
+
693
+ if (chatWindow) {
694
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
695
+ chatWindow.webContents.send('agent-response', {
696
+ text: result.findings?.length > 0
697
+ ? `🔍 Research findings:\n\n${result.findings.join('\n\n')}`
698
+ : `🔍 No findings for query.`,
699
+ type: 'message',
700
+ timestamp: Date.now()
701
+ });
702
+ }
703
+ } catch (error) {
704
+ if (chatWindow) {
705
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
706
+ chatWindow.webContents.send('agent-response', {
707
+ text: `❌ Research failed: ${error.message}`,
708
+ type: 'error',
709
+ timestamp: Date.now()
710
+ });
711
+ }
712
+ }
713
+ return;
714
+ }
715
+
716
+ // /build - Use builder agent
717
+ if (message.startsWith('/build ')) {
718
+ const spec = message.slice('/build '.length).trim();
719
+ if (chatWindow) {
720
+ chatWindow.webContents.send('agent-response', {
721
+ text: `🔨 Starting build: "${spec}"`,
722
+ type: 'system',
723
+ timestamp: Date.now()
724
+ });
725
+ chatWindow.webContents.send('agent-typing', { isTyping: true });
726
+ }
727
+
728
+ try {
729
+ const { orchestrator } = getAgentSystem();
730
+ const result = await orchestrator.build(spec);
731
+
732
+ if (chatWindow) {
733
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
734
+ chatWindow.webContents.send('agent-response', {
735
+ text: `🔨 Build complete:\n\n${JSON.stringify(result, null, 2)}`,
736
+ type: result.status === 'success' ? 'message' : 'error',
737
+ timestamp: Date.now()
738
+ });
739
+ }
740
+ } catch (error) {
741
+ if (chatWindow) {
742
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
743
+ chatWindow.webContents.send('agent-response', {
744
+ text: `❌ Build failed: ${error.message}`,
745
+ type: 'error',
746
+ timestamp: Date.now()
747
+ });
748
+ }
749
+ }
750
+ return;
751
+ }
752
+
753
+ // /verify - Use verifier agent
754
+ if (message.startsWith('/verify ')) {
755
+ const target = message.slice('/verify '.length).trim();
756
+ if (chatWindow) {
757
+ chatWindow.webContents.send('agent-response', {
758
+ text: `✅ Verifying: "${target}"`,
759
+ type: 'system',
760
+ timestamp: Date.now()
761
+ });
762
+ chatWindow.webContents.send('agent-typing', { isTyping: true });
763
+ }
764
+
765
+ try {
766
+ const { orchestrator } = getAgentSystem();
767
+ const result = await orchestrator.verify(target);
768
+
769
+ if (chatWindow) {
770
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
771
+ chatWindow.webContents.send('agent-response', {
772
+ text: `✅ Verification results:\n\n${JSON.stringify(result, null, 2)}`,
773
+ type: result.passed ? 'message' : 'error',
774
+ timestamp: Date.now()
775
+ });
776
+ }
777
+ } catch (error) {
778
+ if (chatWindow) {
779
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
780
+ chatWindow.webContents.send('agent-response', {
781
+ text: `❌ Verification failed: ${error.message}`,
782
+ type: 'error',
783
+ timestamp: Date.now()
784
+ });
785
+ }
786
+ }
787
+ return;
788
+ }
789
+
790
+ // /agent-status - Get multi-agent system status
791
+ if (message === '/agent-status' || message === '/agents') {
792
+ try {
793
+ const { stateManager, orchestrator } = getAgentSystem();
794
+ const state = stateManager.getState();
795
+ const currentSession = orchestrator.currentSession;
796
+
797
+ const statusText = `
798
+ 🤖 **Multi-Agent System Status**
799
+
800
+ **Session:** ${currentSession || 'No active session'}
801
+ **Task Queue:** ${state.taskQueue.length} pending
802
+ **Completed:** ${state.completedTasks.length}
803
+ **Failed:** ${state.failedTasks.length}
804
+ **Handoffs:** ${state.handoffs.length}
805
+
806
+ **Available Commands:**
807
+ • \`/orchestrate <task>\` - Full multi-agent task execution
808
+ • \`/research <query>\` - Research using RLC patterns
809
+ • \`/build <spec>\` - Build code with builder agent
810
+ • \`/verify <target>\` - Verify code/changes
811
+ • \`/agent-reset\` - Reset agent system state
812
+ `;
813
+
814
+ if (chatWindow) {
815
+ chatWindow.webContents.send('agent-response', {
816
+ text: statusText,
817
+ type: 'system',
818
+ timestamp: Date.now()
819
+ });
820
+ }
821
+ } catch (error) {
822
+ if (chatWindow) {
823
+ chatWindow.webContents.send('agent-response', {
824
+ text: `❌ Failed to get status: ${error.message}`,
825
+ type: 'error',
826
+ timestamp: Date.now()
827
+ });
828
+ }
829
+ }
830
+ return;
831
+ }
832
+
833
+ // /agent-reset - Reset multi-agent system
834
+ if (message === '/agent-reset') {
835
+ try {
836
+ const { stateManager } = getAgentSystem();
837
+ stateManager.resetState();
838
+ agentSystem = null;
839
+
840
+ if (chatWindow) {
841
+ chatWindow.webContents.send('agent-response', {
842
+ text: '🔄 Multi-agent system reset successfully.',
843
+ type: 'system',
844
+ timestamp: Date.now()
845
+ });
846
+ }
847
+ } catch (error) {
848
+ if (chatWindow) {
849
+ chatWindow.webContents.send('agent-response', {
850
+ text: `❌ Reset failed: ${error.message}`,
851
+ type: 'error',
852
+ timestamp: Date.now()
853
+ });
854
+ }
855
+ }
856
+ return;
857
+ }
858
+
859
+ let commandResult = aiService.handleCommand(message);
860
+
861
+ // Handle async commands (like /login)
862
+ if (commandResult && typeof commandResult.then === 'function') {
863
+ commandResult = await commandResult;
864
+ }
865
+
866
+ if (commandResult) {
867
+ if (chatWindow) {
868
+ chatWindow.webContents.send('agent-response', {
869
+ text: commandResult.message,
870
+ type: commandResult.type,
871
+ timestamp: Date.now()
872
+ });
873
+ }
874
+ return;
875
+ }
876
+ }
877
+
878
+ // Check if we should include visual context (expanded triggers for agentic actions)
879
+ const includeVisualContext = message.toLowerCase().includes('screen') ||
880
+ message.toLowerCase().includes('see') ||
881
+ message.toLowerCase().includes('look') ||
882
+ message.toLowerCase().includes('show') ||
883
+ message.toLowerCase().includes('capture') ||
884
+ message.toLowerCase().includes('click') ||
885
+ message.toLowerCase().includes('type') ||
886
+ message.toLowerCase().includes('print') ||
887
+ message.toLowerCase().includes('open') ||
888
+ message.toLowerCase().includes('close') ||
889
+ visualContextHistory.length > 0;
890
+
891
+ // Send initial "thinking" indicator
892
+ if (chatWindow) {
893
+ chatWindow.webContents.send('agent-typing', { isTyping: true });
894
+ }
895
+
896
+ try {
897
+ // Call AI service
898
+ const result = await aiService.sendMessage(message, {
899
+ includeVisualContext
900
+ });
901
+
902
+ if (chatWindow) {
903
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
904
+
905
+ if (result.success) {
906
+ // Check if response contains actions
907
+ console.log('[AGENTIC] Parsing response for actions...');
908
+ const actionData = aiService.parseActions(result.message);
909
+ console.log('[AGENTIC] parseActions result:', actionData ? 'found' : 'null');
910
+
911
+ if (actionData && actionData.actions && actionData.actions.length > 0) {
912
+ console.log('[AGENTIC] AI returned actions:', actionData.actions.length);
913
+ console.log('[AGENTIC] Actions:', JSON.stringify(actionData.actions));
914
+
915
+ // Store pending actions
916
+ pendingActions = actionData;
917
+
918
+ // Send response with action data
919
+ chatWindow.webContents.send('agent-response', {
920
+ text: result.message,
921
+ timestamp: Date.now(),
922
+ provider: result.provider,
923
+ hasVisualContext: result.hasVisualContext,
924
+ hasActions: true,
925
+ actionData: actionData
926
+ });
927
+
928
+ // If agentic mode, execute immediately
929
+ if (agenticMode) {
930
+ console.log('[AGENTIC] Auto-executing actions (agentic mode)');
931
+ executeActionsAndRespond(actionData);
932
+ }
933
+ } else {
934
+ console.log('[AGENTIC] No actions detected in response');
935
+ // Normal response without actions
936
+ chatWindow.webContents.send('agent-response', {
937
+ text: result.message,
938
+ timestamp: Date.now(),
939
+ provider: result.provider,
940
+ hasVisualContext: result.hasVisualContext
941
+ });
942
+ }
943
+ } else {
944
+ chatWindow.webContents.send('agent-response', {
945
+ text: `Error: ${result.error}`,
946
+ type: 'error',
947
+ timestamp: Date.now()
948
+ });
949
+ }
950
+ }
951
+ } catch (error) {
952
+ console.error('AI service error:', error);
953
+ if (chatWindow) {
954
+ chatWindow.webContents.send('agent-typing', { isTyping: false });
955
+ chatWindow.webContents.send('agent-response', {
956
+ text: `Error: ${error.message}`,
957
+ type: 'error',
958
+ timestamp: Date.now()
959
+ });
960
+ }
961
+ }
962
+ });
963
+
964
+ // Helper for executing actions with visual feedback and overlay management
965
+ async function performSafeAgenticAction(action) {
966
+ // Only intercept clicks/drags that need overlay interaction
967
+ if (action.type === 'click' || action.type === 'double_click' || action.type === 'right_click' || action.type === 'drag') {
968
+ let x = action.x || action.fromX;
969
+ let y = action.y || action.fromY;
970
+
971
+ // Coordinate Scaling for Precision (Fix for Q4)
972
+ // If visual context exists, scale from Image Space -> Screen Space
973
+ const latestVisual = aiService.getLatestVisualContext();
974
+ if (latestVisual && latestVisual.width && latestVisual.height) {
975
+ const display = screen.getPrimaryDisplay();
976
+ const screenW = display.bounds.width; // e.g., 1920
977
+ const screenH = display.bounds.height; // e.g., 1080
978
+ // Calculate scale multiples
979
+ const scaleX = screenW / latestVisual.width;
980
+ const scaleY = screenH / latestVisual.height;
981
+
982
+ // Only apply if there's a significant difference (e.g. > 1% mismatch)
983
+ if (Math.abs(scaleX - 1) > 0.01 || Math.abs(scaleY - 1) > 0.01) {
984
+ console.log(`[EXECUTOR] Scaling coords from ${latestVisual.width}x${latestVisual.height} to ${screenW}x${screenH} (Target: ${x},${y})`);
985
+ x = Math.round(x * scaleX);
986
+ y = Math.round(y * scaleY);
987
+ // Update action object for system automation
988
+ if(action.x) action.x = x;
989
+ if(action.y) action.y = y;
990
+ if(action.fromX) action.fromX = x;
991
+ if(action.fromY) action.fromY = y;
992
+ if(action.toX) action.toX = Math.round(action.toX * scaleX);
993
+ if(action.toY) action.toY = Math.round(action.toY * scaleY);
994
+ console.log(`[EXECUTOR] Scaled target: ${x},${y}`);
995
+ }
996
+ }
997
+
998
+ console.log(`[EXECUTOR] Intercepting ${action.type} at (${x},${y})`);
999
+
1000
+ // 1. Visual Feedback (Pulse - Doppler Effect)
1001
+ if (overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.webContents) {
1002
+ overlayWindow.webContents.send('overlay-command', {
1003
+ action: 'pulse-click',
1004
+ x: x,
1005
+ y: y,
1006
+ label: action.reason ? 'Action' : undefined
1007
+ });
1008
+ }
1009
+
1010
+ // 2. Wait for user to see pulse (Doppler expansion)
1011
+ await new Promise(r => setTimeout(r, 600));
1012
+
1013
+ // 3. Prepare for Pass-through
1014
+ const wasVisible = overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible();
1015
+ if (wasVisible) {
1016
+ // A. Disable renderer pointer-events (CSS level)
1017
+ // This ensures elements like dots don't capture the click
1018
+ overlayWindow.webContents.send('overlay-command', {
1019
+ action: 'set-click-through',
1020
+ enabled: true
1021
+ });
1022
+
1023
+ // B. Set Electron window to ignore mouse events FULLY (no forwarding)
1024
+ // This ensures the window is completely transparent to the OS mouse subsystem
1025
+ overlayWindow.setIgnoreMouseEvents(true);
1026
+
1027
+ // Give OS time to update window regions
1028
+ await new Promise(r => setTimeout(r, 50));
1029
+ }
1030
+
1031
+ // 4. Exec via System Automation
1032
+ let result;
1033
+ try {
1034
+ result = await aiService.systemAutomation.executeAction(action);
1035
+ } catch (e) {
1036
+ result = { success: false, error: e.message };
1037
+ }
1038
+
1039
+ // 5. Restore Overlay Interactability
1040
+ if (wasVisible && overlayWindow && !overlayWindow.isDestroyed()) {
1041
+ // Brief delay to ensure OS processed the click
1042
+ await new Promise(r => setTimeout(r, 50));
1043
+
1044
+ // A. Restore renderer pointer-events
1045
+ overlayWindow.webContents.send('overlay-command', {
1046
+ action: 'set-click-through',
1047
+ enabled: false
1048
+ });
1049
+
1050
+ // B. Restore Electron window behavior (forwarding enabled for UI interaction)
1051
+ // Note: We use forward: true so users can click dots but see through transparent areas
1052
+ overlayWindow.setIgnoreMouseEvents(true, { forward: true });
1053
+ }
1054
+
1055
+ return result;
1056
+ }
1057
+
1058
+ // Non-spatial actions (type, key, wait) - just execute
1059
+ return aiService.systemAutomation.executeAction(action);
1060
+ }
1061
+
1062
+ // Execute actions and send results
1063
+ async function executeActionsAndRespond(actionData) {
1064
+ if (!chatWindow) return;
1065
+
1066
+ chatWindow.webContents.send('action-executing', {
1067
+ thought: actionData.thought,
1068
+ total: actionData.actions.length
1069
+ });
1070
+
1071
+ try {
1072
+ const results = await aiService.executeActions(
1073
+ actionData,
1074
+ // Progress callback
1075
+ (result, index, total) => {
1076
+ chatWindow.webContents.send('action-progress', {
1077
+ current: index + 1,
1078
+ total,
1079
+ result
1080
+ });
1081
+ },
1082
+ // Screenshot callback - MUST hide overlay before capture
1083
+ async () => {
1084
+ // Hide overlay before capturing so AI sees actual screen
1085
+ const wasOverlayVisible = overlayWindow && overlayWindow.isVisible();
1086
+ if (wasOverlayVisible) {
1087
+ overlayWindow.hide();
1088
+ await new Promise(resolve => setTimeout(resolve, 50));
1089
+ }
1090
+
1091
+ const sources = await require('electron').desktopCapturer.getSources({
1092
+ types: ['screen'],
1093
+ thumbnailSize: {
1094
+ width: screen.getPrimaryDisplay().bounds.width,
1095
+ height: screen.getPrimaryDisplay().bounds.height
1096
+ }
1097
+ });
1098
+
1099
+ // Restore overlay after capture
1100
+ if (wasOverlayVisible && overlayWindow) {
1101
+ overlayWindow.show();
1102
+ }
1103
+
1104
+ if (sources.length > 0) {
1105
+ const imageData = {
1106
+ dataURL: sources[0].thumbnail.toDataURL(),
1107
+ width: sources[0].thumbnail.getSize().width,
1108
+ height: sources[0].thumbnail.getSize().height,
1109
+ timestamp: Date.now()
1110
+ };
1111
+ storeVisualContext(imageData);
1112
+ }
1113
+ },
1114
+ // Options with safe executor
1115
+ { actionExecutor: performSafeAgenticAction }
1116
+ );
1117
+
1118
+ // Send completion notification
1119
+ chatWindow.webContents.send('action-complete', {
1120
+ success: results.success,
1121
+ actionsCount: actionData.actions.length,
1122
+ thought: results.thought,
1123
+ verification: results.verification,
1124
+ results: results.results
1125
+ });
1126
+
1127
+ // If screenshot was requested, capture and show result
1128
+ if (results.screenshotRequested) {
1129
+ await new Promise(resolve => setTimeout(resolve, 500));
1130
+
1131
+ // Hide overlay before capturing
1132
+ const wasOverlayVisible = overlayWindow && overlayWindow.isVisible();
1133
+ if (wasOverlayVisible) {
1134
+ overlayWindow.hide();
1135
+ await new Promise(resolve => setTimeout(resolve, 50));
1136
+ }
1137
+
1138
+ const sources = await require('electron').desktopCapturer.getSources({
1139
+ types: ['screen'],
1140
+ thumbnailSize: {
1141
+ width: screen.getPrimaryDisplay().bounds.width,
1142
+ height: screen.getPrimaryDisplay().bounds.height
1143
+ }
1144
+ });
1145
+
1146
+ // Restore overlay after capture
1147
+ if (wasOverlayVisible && overlayWindow) {
1148
+ overlayWindow.show();
1149
+ }
1150
+
1151
+ if (sources.length > 0) {
1152
+ const imageData = {
1153
+ dataURL: sources[0].thumbnail.toDataURL(),
1154
+ width: sources[0].thumbnail.getSize().width,
1155
+ height: sources[0].thumbnail.getSize().height,
1156
+ timestamp: Date.now()
1157
+ };
1158
+ storeVisualContext(imageData);
1159
+ chatWindow.webContents.send('screen-captured', imageData);
1160
+ }
1161
+ }
1162
+
1163
+ } catch (error) {
1164
+ console.error('[AGENTIC] Action execution error:', error);
1165
+ chatWindow.webContents.send('action-complete', {
1166
+ success: false,
1167
+ actionsCount: actionData.actions ? actionData.actions.length : 0,
1168
+ error: error.message
1169
+ });
1170
+ }
1171
+
1172
+ pendingActions = null;
1173
+ }
1174
+
1175
+ // Handle confirmed action execution
1176
+ ipcMain.on('execute-actions', async (event, actionData) => {
1177
+ console.log('[AGENTIC] User confirmed action execution');
1178
+ await executeActionsAndRespond(actionData || pendingActions);
1179
+ });
1180
+
1181
+ // Handle action cancellation
1182
+ ipcMain.on('cancel-actions', () => {
1183
+ console.log('[AGENTIC] User cancelled actions');
1184
+ pendingActions = null;
1185
+ aiService.clearPendingAction();
1186
+ if (chatWindow) {
1187
+ chatWindow.webContents.send('agent-response', {
1188
+ text: 'Actions cancelled.',
1189
+ type: 'system',
1190
+ timestamp: Date.now()
1191
+ });
1192
+ }
1193
+ });
1194
+
1195
+ // ===== SAFETY GUARDRAILS IPC HANDLERS =====
1196
+
1197
+ // Analyze action safety before execution
1198
+ ipcMain.handle('analyze-action-safety', (event, { action, targetInfo }) => {
1199
+ return aiService.analyzeActionSafety(action, targetInfo || {});
1200
+ });
1201
+
1202
+ // Get pending action awaiting confirmation
1203
+ ipcMain.handle('get-pending-action', () => {
1204
+ return aiService.getPendingAction();
1205
+ });
1206
+
1207
+ // Confirm pending action and resume execution
1208
+ ipcMain.handle('confirm-pending-action', async (event, { actionId }) => {
1209
+ console.log('[SAFETY] User confirmed action:', actionId);
1210
+
1211
+ const pending = aiService.getPendingAction();
1212
+ if (!pending || pending.actionId !== actionId) {
1213
+ return { success: false, error: 'No matching pending action' };
1214
+ }
1215
+
1216
+ // Resume execution after confirmation
1217
+ try {
1218
+ const results = await aiService.resumeAfterConfirmation(
1219
+ // Progress callback
1220
+ (result, index, total) => {
1221
+ if (chatWindow && !chatWindow.isDestroyed()) {
1222
+ chatWindow.webContents.send('action-progress', {
1223
+ current: index + 1,
1224
+ total,
1225
+ result,
1226
+ userConfirmed: true
1227
+ });
1228
+ }
1229
+ },
1230
+ // Screenshot callback
1231
+ async () => {
1232
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1233
+ overlayWindow.hide();
1234
+ }
1235
+ await new Promise(r => setTimeout(r, 100));
1236
+
1237
+ const sources = await desktopCapturer.getSources({
1238
+ types: ['screen'],
1239
+ thumbnailSize: {
1240
+ width: screen.getPrimaryDisplay().bounds.width,
1241
+ height: screen.getPrimaryDisplay().bounds.height
1242
+ }
1243
+ });
1244
+
1245
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1246
+ overlayWindow.show();
1247
+ }
1248
+
1249
+ if (sources.length > 0) {
1250
+ const imageData = {
1251
+ dataURL: sources[0].thumbnail.toDataURL(),
1252
+ width: sources[0].thumbnail.getSize().width,
1253
+ height: sources[0].thumbnail.getSize().height,
1254
+ timestamp: Date.now()
1255
+ };
1256
+ storeVisualContext(imageData);
1257
+ }
1258
+ },
1259
+ // Options with safe executor
1260
+ { actionExecutor: performSafeAgenticAction }
1261
+ );
1262
+
1263
+ // Notify chat of completion
1264
+ if (chatWindow && !chatWindow.isDestroyed()) {
1265
+ chatWindow.webContents.send('action-complete', {
1266
+ success: results.success,
1267
+ userConfirmed: true,
1268
+ results: results.results
1269
+ });
1270
+ }
1271
+
1272
+ return { success: true, results };
1273
+ } catch (error) {
1274
+ console.error('[SAFETY] Resume after confirmation failed:', error);
1275
+ return { success: false, error: error.message };
1276
+ }
1277
+ });
1278
+
1279
+ // Reject pending action
1280
+ ipcMain.handle('reject-pending-action', (event, { actionId }) => {
1281
+ console.log('[SAFETY] User rejected action:', actionId);
1282
+
1283
+ const rejected = aiService.rejectPendingAction(actionId);
1284
+
1285
+ if (rejected && chatWindow && !chatWindow.isDestroyed()) {
1286
+ chatWindow.webContents.send('action-rejected', {
1287
+ actionId,
1288
+ message: 'Action rejected by user'
1289
+ });
1290
+ chatWindow.webContents.send('agent-response', {
1291
+ text: '🛡️ Action rejected. The potentially risky action was not executed.',
1292
+ type: 'system',
1293
+ timestamp: Date.now()
1294
+ });
1295
+ }
1296
+
1297
+ return { success: rejected };
1298
+ });
1299
+
1300
+ // Convert grid label to screen coordinates
1301
+ ipcMain.handle('label-to-coordinates', (event, label) => {
1302
+ // Use gridToPixels from ai-service which uses system-automation
1303
+ const coords = aiService.gridToPixels(label);
1304
+ if (coords) {
1305
+ return {
1306
+ success: true,
1307
+ label,
1308
+ x: coords.x,
1309
+ y: coords.y,
1310
+ screenX: coords.x,
1311
+ screenY: coords.y
1312
+ };
1313
+ }
1314
+ return { success: false, error: `Invalid grid label: ${label}` };
1315
+ });
1316
+
1317
+ // Safe click with overlay hide/show and safety analysis
1318
+ ipcMain.handle('safe-click-at', async (event, { x, y, button = 'left', label, targetInfo }) => {
1319
+ console.log(`[SAFETY] Safe click requested at (${x}, ${y}), button: ${button}`);
1320
+
1321
+ // Analyze safety
1322
+ const action = { type: 'click', x, y, button, reason: label || '' };
1323
+ const safety = aiService.analyzeActionSafety(action, targetInfo || {});
1324
+
1325
+ // If HIGH or CRITICAL, don't execute - require explicit confirmation
1326
+ if (safety.requiresConfirmation) {
1327
+ console.log(`[SAFETY] Click requires confirmation: ${safety.riskLevel}`);
1328
+
1329
+ aiService.setPendingAction({
1330
+ ...safety,
1331
+ actionIndex: 0,
1332
+ remainingActions: [action],
1333
+ completedResults: [],
1334
+ thought: `Click at (${x}, ${y})`,
1335
+ verification: 'Verify click target'
1336
+ });
1337
+
1338
+ // Notify chat window
1339
+ if (chatWindow && !chatWindow.isDestroyed()) {
1340
+ chatWindow.webContents.send('action-requires-confirmation', {
1341
+ actionId: safety.actionId,
1342
+ action: action,
1343
+ safety: safety,
1344
+ description: safety.description,
1345
+ riskLevel: safety.riskLevel,
1346
+ warnings: safety.warnings
1347
+ });
1348
+ }
1349
+
1350
+ return {
1351
+ success: false,
1352
+ pending: true,
1353
+ actionId: safety.actionId,
1354
+ riskLevel: safety.riskLevel,
1355
+ message: `Action requires confirmation: ${safety.warnings.join(', ')}`
1356
+ };
1357
+ }
1358
+
1359
+ // SAFE/LOW/MEDIUM - execute with visual feedback
1360
+ try {
1361
+ // INJECTION: Ensure visual feedback system is loaded
1362
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1363
+ try {
1364
+ const isLoaded = await overlayWindow.webContents.executeJavaScript('window.hasPulseSystem === true').catch(() => false);
1365
+
1366
+ if (!isLoaded) {
1367
+ const css = `
1368
+ .pulse-ring {
1369
+ position: absolute;
1370
+ border-radius: 50%;
1371
+ pointer-events: none;
1372
+ animation: pulse-animation 0.8s ease-out forwards;
1373
+ border: 2px solid #00ffcc;
1374
+ background: radial-gradient(circle, rgba(0,255,204,0.3) 0%, rgba(0,255,204,0) 70%);
1375
+ box-shadow: 0 0 15px rgba(0, 255, 204, 0.6);
1376
+ z-index: 2147483647;
1377
+ transform: translate(-50%, -50%);
1378
+ }
1379
+ @keyframes pulse-animation {
1380
+ 0% { width: 10px; height: 10px; opacity: 1; transform: translate(-50%, -50%) scale(1); }
1381
+ 100% { width: 100px; height: 100px; opacity: 0; transform: translate(-50%, -50%) scale(1.5); }
1382
+ }
1383
+ `;
1384
+ await overlayWindow.webContents.insertCSS(css);
1385
+ overlayWindow.webContents.executeJavaScript(`
1386
+ const { ipcRenderer } = require('electron');
1387
+ window.showPulseClick = (x, y) => {
1388
+ const el = document.createElement('div');
1389
+ el.className = 'pulse-ring';
1390
+ el.style.left = x + 'px';
1391
+ el.style.top = y + 'px';
1392
+ document.body.appendChild(el);
1393
+ setTimeout(() => el.remove(), 1000);
1394
+ };
1395
+ ipcRenderer.removeAllListeners('overlay-command');
1396
+ ipcRenderer.on('overlay-command', (event, data) => {
1397
+ if (data.action === 'pulse-click') window.showPulseClick(data.x, data.y);
1398
+ });
1399
+ window.hasPulseSystem = true;
1400
+ `);
1401
+ }
1402
+ } catch(e) { console.error('Safe click injection error:', e); }
1403
+ }
1404
+
1405
+ // Show visual indicator on overlay
1406
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1407
+ overlayWindow.webContents.send('overlay-command', {
1408
+ action: 'pulse-click', // Updated to pulse
1409
+ x, y,
1410
+ label: label || `${x},${y}`
1411
+ });
1412
+ }
1413
+
1414
+ await new Promise(r => setTimeout(r, 150));
1415
+
1416
+ // Hide overlay for click-through
1417
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1418
+ overlayWindow.hide();
1419
+ }
1420
+
1421
+ await new Promise(r => setTimeout(r, 50));
1422
+
1423
+ // Execute click via system-automation
1424
+ const result = await aiService.systemAutomation.executeAction({
1425
+ type: 'click',
1426
+ x: Math.round(x),
1427
+ y: Math.round(y),
1428
+ button
1429
+ });
1430
+
1431
+ await new Promise(r => setTimeout(r, 100));
1432
+
1433
+ // Restore overlay
1434
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1435
+ overlayWindow.show();
1436
+ }
1437
+
1438
+ console.log(`[SAFETY] Click executed: ${result.success}`);
1439
+
1440
+ return {
1441
+ success: result.success,
1442
+ x, y,
1443
+ riskLevel: safety.riskLevel,
1444
+ error: result.error
1445
+ };
1446
+
1447
+ } catch (error) {
1448
+ console.error('[SAFETY] Safe click failed:', error);
1449
+
1450
+ // Always restore overlay on error
1451
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1452
+ overlayWindow.show();
1453
+ }
1454
+
1455
+ return { success: false, error: error.message };
1456
+ }
1457
+ });
1458
+
1459
+ // ===== WINDOW CONTROLS =====
1460
+ ipcMain.on('minimize-chat', () => {
1461
+ if (chatWindow) {
1462
+ chatWindow.minimize();
1463
+ }
1464
+ });
1465
+
1466
+ ipcMain.on('hide-chat', () => {
1467
+ if (chatWindow) {
1468
+ chatWindow.hide();
1469
+ isChatVisible = false;
1470
+ }
1471
+ });
1472
+
1473
+ // ===== SCREEN CAPTURE (AI Visual Awareness) =====
1474
+ // CRITICAL: Hide overlay before capture so AI sees actual screen content without dots
1475
+ ipcMain.on('capture-screen', async (event, options = {}) => {
1476
+ try {
1477
+ // Hide overlay BEFORE capturing so screenshot shows actual screen (not dots)
1478
+ const wasOverlayVisible = overlayWindow && overlayWindow.isVisible();
1479
+ if (wasOverlayVisible) {
1480
+ overlayWindow.hide();
1481
+ // Brief delay to ensure overlay is fully hidden
1482
+ await new Promise(resolve => setTimeout(resolve, 50));
1483
+ }
1484
+
1485
+ const sources = await desktopCapturer.getSources({
1486
+ types: ['screen'],
1487
+ thumbnailSize: {
1488
+ width: screen.getPrimaryDisplay().bounds.width,
1489
+ height: screen.getPrimaryDisplay().bounds.height
1490
+ }
1491
+ });
1492
+
1493
+ // Restore overlay after capture
1494
+ if (wasOverlayVisible && overlayWindow) {
1495
+ overlayWindow.show();
1496
+ }
1497
+
1498
+ if (sources.length > 0) {
1499
+ const primarySource = sources[0];
1500
+ const thumbnail = primarySource.thumbnail;
1501
+
1502
+ // Get image data
1503
+ const imageData = {
1504
+ dataURL: thumbnail.toDataURL(),
1505
+ width: thumbnail.getSize().width,
1506
+ height: thumbnail.getSize().height,
1507
+ x: 0,
1508
+ y: 0,
1509
+ timestamp: Date.now(),
1510
+ sourceId: primarySource.id,
1511
+ sourceName: primarySource.name
1512
+ };
1513
+
1514
+ // Send to chat window
1515
+ if (chatWindow) {
1516
+ chatWindow.webContents.send('screen-captured', imageData);
1517
+ }
1518
+
1519
+ // Log for debugging
1520
+ console.log(`Screen captured: ${imageData.width}x${imageData.height} (overlay was ${wasOverlayVisible ? 'hidden' : 'already hidden'})`);
1521
+
1522
+ // Store in visual context for AI processing
1523
+ storeVisualContext(imageData);
1524
+ }
1525
+ } catch (error) {
1526
+ console.error('Screen capture failed:', error);
1527
+ // Ensure overlay is restored on error
1528
+ if (overlayWindow && !overlayWindow.isVisible()) {
1529
+ overlayWindow.show();
1530
+ }
1531
+ if (chatWindow) {
1532
+ chatWindow.webContents.send('screen-captured', { error: error.message });
1533
+ }
1534
+ }
1535
+ });
1536
+
1537
+ // Capture a specific region
1538
+ ipcMain.on('capture-region', async (event, { x, y, width, height }) => {
1539
+ try {
1540
+ // Hide overlay BEFORE capturing
1541
+ const wasOverlayVisible = overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible();
1542
+ if (wasOverlayVisible) {
1543
+ overlayWindow.hide();
1544
+ await new Promise(resolve => setTimeout(resolve, 50));
1545
+ }
1546
+
1547
+ const sources = await desktopCapturer.getSources({
1548
+ types: ['screen'],
1549
+ thumbnailSize: {
1550
+ width: screen.getPrimaryDisplay().bounds.width,
1551
+ height: screen.getPrimaryDisplay().bounds.height
1552
+ }
1553
+ });
1554
+
1555
+ // Restore overlay after capture
1556
+ if (wasOverlayVisible && overlayWindow) {
1557
+ overlayWindow.show();
1558
+ }
1559
+
1560
+ if (sources.length > 0) {
1561
+ const primarySource = sources[0];
1562
+ const thumbnail = primarySource.thumbnail;
1563
+
1564
+ // Crop to region
1565
+ const cropped = thumbnail.crop({
1566
+ x: Math.max(0, x),
1567
+ y: Math.max(0, y),
1568
+ width: Math.min(width, thumbnail.getSize().width - x),
1569
+ height: Math.min(height, thumbnail.getSize().height - y)
1570
+ });
1571
+
1572
+ const imageData = {
1573
+ dataURL: cropped.toDataURL(),
1574
+ width: cropped.getSize().width,
1575
+ height: cropped.getSize().height,
1576
+ x,
1577
+ y,
1578
+ timestamp: Date.now(),
1579
+ type: 'region'
1580
+ };
1581
+
1582
+ if (chatWindow) {
1583
+ chatWindow.webContents.send('screen-captured', imageData);
1584
+ }
1585
+
1586
+ storeVisualContext(imageData);
1587
+ }
1588
+ } catch (error) {
1589
+ console.error('Region capture failed:', error);
1590
+ // Ensure overlay is restored on error
1591
+ if (overlayWindow && !overlayWindow.isVisible()) {
1592
+ overlayWindow.show();
1593
+ }
1594
+ }
1595
+ });
1596
+
1597
+ // Get current state
1598
+ ipcMain.handle('get-state', () => {
1599
+ const aiStatus = aiService.getStatus();
1600
+ return {
1601
+ overlayMode,
1602
+ isChatVisible,
1603
+ visualContextCount: visualContextHistory.length,
1604
+ aiProvider: aiStatus.provider,
1605
+ model: aiStatus.model,
1606
+ aiStatus,
1607
+ // Inspect mode state
1608
+ inspectMode: inspectService.isInspectModeActive(),
1609
+ inspectRegionCount: inspectService.getRegions().length
1610
+ };
1611
+ });
1612
+
1613
+ // ===== INSPECT MODE IPC HANDLERS =====
1614
+
1615
+ // Toggle inspect mode
1616
+ ipcMain.on('toggle-inspect-mode', () => {
1617
+ const newState = !inspectService.isInspectModeActive();
1618
+ inspectService.setInspectMode(newState);
1619
+ console.log(`[INSPECT] Mode toggled: ${newState}`);
1620
+
1621
+ // Notify overlay
1622
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1623
+ overlayWindow.webContents.send('inspect-mode-changed', newState);
1624
+ }
1625
+
1626
+ // Notify chat
1627
+ if (chatWindow && !chatWindow.isDestroyed()) {
1628
+ chatWindow.webContents.send('inspect-mode-changed', newState);
1629
+ }
1630
+
1631
+ // If enabled, trigger region detection
1632
+ if (newState) {
1633
+ detectAndSendInspectRegions().catch(err => {
1634
+ console.error('[INSPECT] Region detection failed:', err);
1635
+ });
1636
+ }
1637
+ });
1638
+
1639
+ // Request inspect regions detection
1640
+ ipcMain.on('request-inspect-regions', async () => {
1641
+ await detectAndSendInspectRegions().catch(err => {
1642
+ console.error('[INSPECT] Region detection request failed:', err);
1643
+ });
1644
+ });
1645
+
1646
+ // Handle inspect region selection from overlay
1647
+ ipcMain.on('inspect-region-selected', (event, data) => {
1648
+ console.log('[INSPECT] Region selected:', data);
1649
+
1650
+ // Record the action
1651
+ const trace = inspectService.recordAction({
1652
+ type: 'select',
1653
+ targetId: data.targetId,
1654
+ x: data.x,
1655
+ y: data.y
1656
+ }, data.targetId);
1657
+
1658
+ // Forward to chat window with targetId for AI targeting
1659
+ if (chatWindow && !chatWindow.isDestroyed()) {
1660
+ chatWindow.webContents.send('inspect-region-selected', {
1661
+ ...data,
1662
+ actionId: trace.actionId
1663
+ });
1664
+ }
1665
+
1666
+ // Select the region in service
1667
+ inspectService.selectRegion(data.targetId);
1668
+ });
1669
+
1670
+ // Get inspect context for AI
1671
+ ipcMain.handle('get-inspect-context', () => {
1672
+ return inspectService.generateAIContext();
1673
+ });
1674
+
1675
+ // Get inspect regions
1676
+ ipcMain.handle('get-inspect-regions', () => {
1677
+ return inspectService.getRegions();
1678
+ });
1679
+
1680
+ // Get window context
1681
+ ipcMain.handle('get-window-context', async () => {
1682
+ return await inspectService.updateWindowContext();
1683
+ });
1684
+
1685
+ /**
1686
+ * Detect UI regions and send to overlay
1687
+ */
1688
+ async function detectAndSendInspectRegions() {
1689
+ try {
1690
+ console.log('[INSPECT] Detecting regions...');
1691
+ const results = await inspectService.detectRegions();
1692
+
1693
+ // Send regions to overlay
1694
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1695
+ overlayWindow.webContents.send('inspect-regions-update', results.regions);
1696
+ }
1697
+
1698
+ // Notify chat of new context
1699
+ if (chatWindow && !chatWindow.isDestroyed()) {
1700
+ chatWindow.webContents.send('inspect-context-update', {
1701
+ regionCount: results.regions.length,
1702
+ windowContext: results.windowContext
1703
+ });
1704
+ }
1705
+
1706
+ console.log(`[INSPECT] Detected ${results.regions.length} regions`);
1707
+ return results;
1708
+ } catch (error) {
1709
+ console.error('[INSPECT] Detection failed:', error);
1710
+ return { regions: [], error: error.message };
1711
+ }
1712
+ }
1713
+
1714
+ // ===== AI CLICK-THROUGH AUTOMATION (Q4 FIX) =====
1715
+ // This allows AI to click at coordinates THROUGH the overlay to the background app
1716
+ // The overlay should NOT intercept these programmatic clicks
1717
+ ipcMain.handle('click-through-at', async (event, { x, y, button = 'left', label }) => {
1718
+ try {
1719
+ console.log(`[CLICK-THROUGH] Executing click at (${x}, ${y}) label=${label || 'none'}`);
1720
+
1721
+ // INJECTION: Ensure visual feedback system is loaded on first click
1722
+ if (overlayWindow && !overlayWindow.isDestroyed()) {
1723
+ try {
1724
+ // Check if pulse system is loaded in renderer
1725
+ const isLoaded = await overlayWindow.webContents.executeJavaScript('window.hasPulseSystem === true').catch(() => false);
1726
+
1727
+ if (!isLoaded) {
1728
+ console.log('[CLICK-THROUGH] Injecting visual feedback system...');
1729
+ const css = `
1730
+ .pulse-ring {
1731
+ position: absolute;
1732
+ border-radius: 50%;
1733
+ pointer-events: none;
1734
+ animation: pulse-animation 0.8s ease-out forwards;
1735
+ border: 2px solid #00ffcc;
1736
+ background: radial-gradient(circle, rgba(0,255,204,0.3) 0%, rgba(0,255,204,0) 70%);
1737
+ box-shadow: 0 0 15px rgba(0, 255, 204, 0.6);
1738
+ z-index: 2147483647;
1739
+ transform: translate(-50%, -50%);
1740
+ }
1741
+ @keyframes pulse-animation {
1742
+ 0% { width: 10px; height: 10px; opacity: 1; transform: translate(-50%, -50%) scale(1); }
1743
+ 100% { width: 100px; height: 100px; opacity: 0; transform: translate(-50%, -50%) scale(1.5); }
1744
+ }
1745
+ `;
1746
+ await overlayWindow.webContents.insertCSS(css);
1747
+
1748
+ const js = `
1749
+ const { ipcRenderer } = require('electron');
1750
+ window.showPulseClick = (x, y) => {
1751
+ const el = document.createElement('div');
1752
+ el.className = 'pulse-ring';
1753
+ el.style.left = x + 'px';
1754
+ el.style.top = y + 'px';
1755
+ document.body.appendChild(el);
1756
+ setTimeout(() => el.remove(), 1000);
1757
+ };
1758
+ ipcRenderer.removeAllListeners('overlay-command');
1759
+ ipcRenderer.on('overlay-command', (event, data) => {
1760
+ if (data.action === 'pulse-click') window.showPulseClick(data.x, data.y);
1761
+ });
1762
+ window.hasPulseSystem = true;
1763
+ `;
1764
+ await overlayWindow.webContents.executeJavaScript(js);
1765
+ }
1766
+ } catch (e) { console.error('Visual injection error:', e); }
1767
+ }
1768
+
1769
+ // 1. Show visual feedback on overlay (optional - for user awareness)
1770
+ if (overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.webContents) {
1771
+ overlayWindow.webContents.send('overlay-command', {
1772
+ action: 'pulse-click', // Changed from highlight-coordinate to specific pulse-click
1773
+ x, y, label
1774
+ });
1775
+ }
1776
+
1777
+ // 2. Brief delay for visual feedback (increased to let pulse show)
1778
+ await new Promise(resolve => setTimeout(resolve, 300));
1779
+
1780
+ // 3. Hide overlay to ensure click goes through
1781
+ const wasVisible = overlayWindow && !overlayWindow.isDestroyed() && overlayWindow.isVisible();
1782
+ if (wasVisible) {
1783
+ overlayWindow.hide();
1784
+ // Give Windows DWM more time to process transparency
1785
+ await new Promise(resolve => setTimeout(resolve, 150));
1786
+ }
1787
+
1788
+ // 4. Execute the click using robotjs or similar automation
1789
+ // Note: This requires robotjs to be installed and working
1790
+ try {
1791
+ const robot = require('robotjs');
1792
+ // Double move to ensure OS registers cursor position
1793
+ robot.moveMouse(x, y);
1794
+ robot.moveMouse(x, y);
1795
+ await new Promise(resolve => setTimeout(resolve, 50));
1796
+ robot.mouseClick(button);
1797
+ console.log(`[CLICK-THROUGH] Click executed successfully at (${x}, ${y})`);
1798
+ } catch (robotError) {
1799
+ console.error('[CLICK-THROUGH] Robot click failed:', robotError.message);
1800
+ // Fallback: try using PowerShell on Windows
1801
+ if (process.platform === 'win32') {
1802
+ const { exec } = require('child_process');
1803
+ const psCommand = `
1804
+ Add-Type -AssemblyName System.Windows.Forms
1805
+ [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})
1806
+ Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo);' -Name U32 -Namespace W
1807
+ [W.U32]::mouse_event(0x02, 0, 0, 0, 0)
1808
+ [W.U32]::mouse_event(0x04, 0, 0, 0, 0)
1809
+ `;
1810
+ await new Promise((resolve, reject) => {
1811
+ exec(`powershell -Command "${psCommand.replace(/"/g, '\\"')}"`, (error) => {
1812
+ if (error) reject(error);
1813
+ else resolve();
1814
+ });
1815
+ });
1816
+ console.log(`[CLICK-THROUGH] PowerShell click executed at (${x}, ${y})`);
1817
+ } else {
1818
+ throw robotError;
1819
+ }
1820
+ }
1821
+
1822
+ // 5. Restore overlay after a delay (let the click register)
1823
+ await new Promise(resolve => setTimeout(resolve, 150));
1824
+ if (wasVisible && overlayWindow && !overlayWindow.isDestroyed()) {
1825
+ overlayWindow.show();
1826
+ }
1827
+
1828
+ return { success: true, x, y, label };
1829
+ } catch (error) {
1830
+ console.error('[CLICK-THROUGH] Error:', error);
1831
+ // Ensure overlay is restored on error
1832
+ if (overlayWindow && !overlayWindow.isDestroyed() && !overlayWindow.isVisible()) {
1833
+ overlayWindow.show();
1834
+ }
1835
+ return { success: false, error: error.message };
1836
+ }
1837
+ });
1838
+
1839
+ // NOTE: label-to-coordinates, analyze-action-safety, safe-click-at, confirm-pending-action,
1840
+ // reject-pending-action, and get-pending-action handlers are registered above in
1841
+ // SAFETY GUARDRAILS IPC HANDLERS section. Do NOT register duplicate handlers here.
1842
+
1843
+ // NOTE: strict mode requires unique IPC handlers
1844
+ // Previously duplicate handlers were removed from here.
1845
+
1846
+ // Set AI provider
1847
+ ipcMain.on('set-ai-provider', (event, provider) => {
1848
+ const success = aiService.setProvider(provider);
1849
+ if (chatWindow) {
1850
+ chatWindow.webContents.send('provider-changed', {
1851
+ provider,
1852
+ success,
1853
+ status: aiService.getStatus()
1854
+ });
1855
+ }
1856
+ });
1857
+
1858
+ // Set API key
1859
+ ipcMain.on('set-api-key', (event, { provider, key }) => {
1860
+ const success = aiService.setApiKey(provider, key);
1861
+ if (chatWindow) {
1862
+ chatWindow.webContents.send('api-key-set', { provider, success });
1863
+ }
1864
+ });
1865
+
1866
+ // Check auth status for a provider
1867
+ ipcMain.on('check-auth', async (event, provider) => {
1868
+ const status = aiService.getStatus();
1869
+ const currentProvider = provider || status.provider;
1870
+ let authStatus = 'disconnected';
1871
+
1872
+ if (currentProvider === 'copilot') {
1873
+ // Check if Copilot token exists
1874
+ const tokenPath = require('path').join(app.getPath('appData'), 'copilot-agent', 'copilot-token.json');
1875
+ try {
1876
+ if (require('fs').existsSync(tokenPath)) {
1877
+ authStatus = 'connected';
1878
+ }
1879
+ } catch (e) {
1880
+ authStatus = 'disconnected';
1881
+ }
1882
+ } else if (currentProvider === 'ollama') {
1883
+ // Ollama doesn't need auth, just check if running
1884
+ authStatus = 'connected';
1885
+ } else {
1886
+ // OpenAI/Anthropic need API keys
1887
+ authStatus = status.hasApiKey ? 'connected' : 'disconnected';
1888
+ }
1889
+
1890
+ if (chatWindow) {
1891
+ chatWindow.webContents.send('auth-status', {
1892
+ provider: currentProvider,
1893
+ status: authStatus
1894
+ });
1895
+ }
1896
+ });
1897
+
1898
+ // ===== VISUAL AWARENESS =====
1899
+
1900
+ // Get active window info
1901
+ ipcMain.handle('get-active-window', async () => {
1902
+ return await visualAwareness.getActiveWindow();
1903
+ });
1904
+
1905
+ // Find element at coordinates
1906
+ ipcMain.handle('find-element-at', async (event, { x, y }) => {
1907
+ return await visualAwareness.findElementAtPoint(x, y);
1908
+ });
1909
+
1910
+ // Detect UI elements
1911
+ ipcMain.handle('detect-ui-elements', async (event, options = {}) => {
1912
+ return await visualAwareness.detectUIElements(options);
1913
+ });
1914
+
1915
+ // Extract text via OCR
1916
+ ipcMain.handle('extract-text', async (event, options = {}) => {
1917
+ const latestContext = visualContextHistory[visualContextHistory.length - 1];
1918
+ if (!latestContext) {
1919
+ return { error: 'No screen capture available. Capture screen first.' };
1920
+ }
1921
+ return await visualAwareness.extractTextFromImage(latestContext, options);
1922
+ });
1923
+
1924
+ // Full screen analysis
1925
+ ipcMain.handle('analyze-screen', async (event, options = {}) => {
1926
+ const latestContext = visualContextHistory[visualContextHistory.length - 1];
1927
+ if (!latestContext) {
1928
+ return { error: 'No screen capture available. Capture screen first.' };
1929
+ }
1930
+ const analysis = await visualAwareness.analyzeScreen(latestContext, options);
1931
+
1932
+ // Send analysis to chat window
1933
+ if (chatWindow) {
1934
+ chatWindow.webContents.send('screen-analysis', analysis);
1935
+ }
1936
+
1937
+ return analysis;
1938
+ });
1939
+
1940
+ // Get screen diff history
1941
+ ipcMain.handle('get-screen-diff-history', () => {
1942
+ return visualAwareness.getScreenDiffHistory();
1943
+ });
1944
+
1945
+ // ===== MULTI-AGENT SYSTEM IPC HANDLERS =====
1946
+ // Initialize agent system lazily
1947
+ let agentSystem = null;
1948
+
1949
+ function getAgentSystem() {
1950
+ if (!agentSystem) {
1951
+ agentSystem = createAgentSystem(aiService);
1952
+ }
1953
+ return agentSystem;
1954
+ }
1955
+
1956
+ // Spawn a new agent session
1957
+ ipcMain.handle('agent-spawn', async (event, { task, options = {} }) => {
1958
+ try {
1959
+ const { orchestrator } = getAgentSystem();
1960
+ const sessionId = await orchestrator.startSession(task);
1961
+
1962
+ if (chatWindow && !chatWindow.isDestroyed()) {
1963
+ chatWindow.webContents.send('agent-event', {
1964
+ type: 'session-started',
1965
+ sessionId,
1966
+ task,
1967
+ timestamp: Date.now()
1968
+ });
1969
+ }
1970
+
1971
+ return { success: true, sessionId };
1972
+ } catch (error) {
1973
+ console.error('[AGENT] Spawn failed:', error);
1974
+ return { success: false, error: error.message };
1975
+ }
1976
+ });
1977
+
1978
+ // Execute a task with the agent system
1979
+ ipcMain.handle('agent-run', async (event, { task, options = {} }) => {
1980
+ try {
1981
+ const { orchestrator } = getAgentSystem();
1982
+
1983
+ // Notify chat of execution start
1984
+ if (chatWindow && !chatWindow.isDestroyed()) {
1985
+ chatWindow.webContents.send('agent-event', {
1986
+ type: 'execution-started',
1987
+ task,
1988
+ timestamp: Date.now()
1989
+ });
1990
+ }
1991
+
1992
+ const result = await orchestrator.orchestrate(task);
1993
+
1994
+ // Notify chat of completion
1995
+ if (chatWindow && !chatWindow.isDestroyed()) {
1996
+ chatWindow.webContents.send('agent-event', {
1997
+ type: 'execution-complete',
1998
+ task,
1999
+ result,
2000
+ timestamp: Date.now()
2001
+ });
2002
+ }
2003
+
2004
+ return { success: true, result };
2005
+ } catch (error) {
2006
+ console.error('[AGENT] Run failed:', error);
2007
+
2008
+ if (chatWindow && !chatWindow.isDestroyed()) {
2009
+ chatWindow.webContents.send('agent-event', {
2010
+ type: 'execution-error',
2011
+ task,
2012
+ error: error.message,
2013
+ timestamp: Date.now()
2014
+ });
2015
+ }
2016
+
2017
+ return { success: false, error: error.message };
2018
+ }
2019
+ });
2020
+
2021
+ // Research a topic using the researcher agent
2022
+ ipcMain.handle('agent-research', async (event, { query, options = {} }) => {
2023
+ try {
2024
+ const { orchestrator } = getAgentSystem();
2025
+ const result = await orchestrator.research(query);
2026
+ return { success: true, result };
2027
+ } catch (error) {
2028
+ console.error('[AGENT] Research failed:', error);
2029
+ return { success: false, error: error.message };
2030
+ }
2031
+ });
2032
+
2033
+ // Verify code/changes using the verifier agent
2034
+ ipcMain.handle('agent-verify', async (event, { target, options = {} }) => {
2035
+ try {
2036
+ const { orchestrator } = getAgentSystem();
2037
+ const result = await orchestrator.verify(target);
2038
+ return { success: true, result };
2039
+ } catch (error) {
2040
+ console.error('[AGENT] Verify failed:', error);
2041
+ return { success: false, error: error.message };
2042
+ }
2043
+ });
2044
+
2045
+ // Build code/features using the builder agent
2046
+ ipcMain.handle('agent-build', async (event, { specification, options = {} }) => {
2047
+ try {
2048
+ const { orchestrator } = getAgentSystem();
2049
+ const result = await orchestrator.build(specification);
2050
+ return { success: true, result };
2051
+ } catch (error) {
2052
+ console.error('[AGENT] Build failed:', error);
2053
+ return { success: false, error: error.message };
2054
+ }
2055
+ });
2056
+
2057
+ // Get agent system status
2058
+ ipcMain.handle('agent-status', async () => {
2059
+ try {
2060
+ const { stateManager, orchestrator } = getAgentSystem();
2061
+ const state = stateManager.getState();
2062
+ const currentSession = orchestrator.currentSession;
2063
+
2064
+ return {
2065
+ success: true,
2066
+ status: {
2067
+ initialized: !!agentSystem,
2068
+ currentSession,
2069
+ taskQueue: state.taskQueue.length,
2070
+ completedTasks: state.completedTasks.length,
2071
+ failedTasks: state.failedTasks.length,
2072
+ activeAgents: Object.keys(state.agents).filter(k => state.agents[k].currentTask).length,
2073
+ handoffCount: state.handoffs.length,
2074
+ sessions: state.sessions
2075
+ }
2076
+ };
2077
+ } catch (error) {
2078
+ console.error('[AGENT] Status failed:', error);
2079
+ return { success: false, error: error.message };
2080
+ }
2081
+ });
2082
+
2083
+ // Reset agent system state
2084
+ ipcMain.handle('agent-reset', async () => {
2085
+ try {
2086
+ const { stateManager } = getAgentSystem();
2087
+ stateManager.resetState();
2088
+ agentSystem = null; // Force re-initialization
2089
+
2090
+ return { success: true, message: 'Agent system reset successfully' };
2091
+ } catch (error) {
2092
+ console.error('[AGENT] Reset failed:', error);
2093
+ return { success: false, error: error.message };
2094
+ }
2095
+ });
2096
+
2097
+ // Get agent handoff history
2098
+ ipcMain.handle('agent-handoffs', async () => {
2099
+ try {
2100
+ const { stateManager } = getAgentSystem();
2101
+ const state = stateManager.getState();
2102
+ return { success: true, handoffs: state.handoffs };
2103
+ } catch (error) {
2104
+ console.error('[AGENT] Get handoffs failed:', error);
2105
+ return { success: false, error: error.message };
2106
+ }
2107
+ });
2108
+ }
2109
+
2110
+ // ===== VISUAL CONTEXT MANAGEMENT (AI Awareness) =====
2111
+ let visualContextHistory = [];
2112
+ const MAX_VISUAL_CONTEXT_ITEMS = 10;
2113
+
2114
+ /**
2115
+ * Store visual context for AI processing
2116
+ */
2117
+ function storeVisualContext(imageData) {
2118
+ visualContextHistory.push({
2119
+ ...imageData,
2120
+ id: `vc-${Date.now()}`
2121
+ });
2122
+
2123
+ // Keep only recent items
2124
+ if (visualContextHistory.length > MAX_VISUAL_CONTEXT_ITEMS) {
2125
+ visualContextHistory.shift();
2126
+ }
2127
+
2128
+ // Also add to AI service for vision capabilities
2129
+ aiService.addVisualContext(imageData);
2130
+
2131
+ // Notify chat window of visual context update
2132
+ if (chatWindow) {
2133
+ chatWindow.webContents.send('visual-context-update', {
2134
+ count: visualContextHistory.length,
2135
+ latest: imageData.timestamp
2136
+ });
2137
+ }
2138
+ }
2139
+
2140
+ /**
2141
+ * Initialize the application
2142
+ */
2143
+ app.whenReady().then(() => {
2144
+ loadChatBoundsPrefs();
2145
+ createOverlayWindow();
2146
+ createChatWindow();
2147
+ createTray();
2148
+ registerShortcuts();
2149
+ setupIPC();
2150
+
2151
+ // Set up Copilot OAuth callback to notify chat on auth completion
2152
+ aiService.setOAuthCallback((result) => {
2153
+ if (chatWindow && !chatWindow.isDestroyed()) {
2154
+ chatWindow.webContents.send('agent-response', {
2155
+ text: result.success ? result.message : `Authentication failed: ${result.message}`,
2156
+ type: result.success ? 'system' : 'error',
2157
+ timestamp: Date.now()
2158
+ });
2159
+
2160
+ // Also send auth status update
2161
+ chatWindow.webContents.send('auth-status', {
2162
+ provider: 'copilot',
2163
+ status: result.success ? 'connected' : 'error'
2164
+ });
2165
+ }
2166
+ });
2167
+
2168
+ // Try to load saved Copilot token
2169
+ aiService.loadCopilotToken();
2170
+
2171
+ // Send initial auth status after a short delay (wait for chat window to be ready)
2172
+ setTimeout(() => {
2173
+ if (chatWindow && !chatWindow.isDestroyed()) {
2174
+ const status = aiService.getStatus();
2175
+ const tokenPath = require('path').join(app.getPath('appData'), 'copilot-agent', 'copilot-token.json');
2176
+ const hasCopilotToken = require('fs').existsSync(tokenPath);
2177
+
2178
+ chatWindow.webContents.send('auth-status', {
2179
+ provider: status.provider,
2180
+ status: hasCopilotToken ? 'connected' : 'disconnected'
2181
+ });
2182
+ }
2183
+ }, 1000);
2184
+
2185
+ app.on('activate', () => {
2186
+ if (BrowserWindow.getAllWindows().length === 0) {
2187
+ createOverlayWindow();
2188
+ createChatWindow();
2189
+ }
2190
+ });
2191
+ });
2192
+
2193
+ // Quit when all windows are closed (except on macOS)
2194
+ app.on('window-all-closed', () => {
2195
+ if (process.platform !== 'darwin') {
2196
+ app.quit();
2197
+ }
2198
+ });
2199
+
2200
+ // Clean up shortcuts on quit
2201
+ app.on('will-quit', () => {
2202
+ globalShortcut.unregisterAll();
2203
+ });
2204
+
2205
+ // Prevent app from quitting when closing chat window
2206
+ app.on('before-quit', () => {
2207
+ app.isQuitting = true;
2208
+ });