copilot-liku-cli 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/ARCHITECTURE.md +411 -0
  2. package/CONFIGURATION.md +302 -0
  3. package/CONTRIBUTING.md +225 -0
  4. package/ELECTRON_README.md +121 -0
  5. package/INSTALLATION.md +350 -0
  6. package/LICENSE.md +1 -0
  7. package/PROJECT_STATUS.md +229 -0
  8. package/QUICKSTART.md +255 -0
  9. package/README.md +167 -0
  10. package/TESTING.md +274 -0
  11. package/package.json +61 -0
  12. package/scripts/start.js +30 -0
  13. package/src/assets/tray-icon.png +0 -0
  14. package/src/cli/commands/agent.js +327 -0
  15. package/src/cli/commands/click.js +108 -0
  16. package/src/cli/commands/drag.js +85 -0
  17. package/src/cli/commands/find.js +109 -0
  18. package/src/cli/commands/keys.js +132 -0
  19. package/src/cli/commands/mouse.js +79 -0
  20. package/src/cli/commands/repl.js +290 -0
  21. package/src/cli/commands/screenshot.js +72 -0
  22. package/src/cli/commands/scroll.js +74 -0
  23. package/src/cli/commands/start.js +67 -0
  24. package/src/cli/commands/type.js +57 -0
  25. package/src/cli/commands/wait.js +84 -0
  26. package/src/cli/commands/window.js +104 -0
  27. package/src/cli/liku.js +249 -0
  28. package/src/cli/util/output.js +174 -0
  29. package/src/main/agents/base-agent.js +410 -0
  30. package/src/main/agents/builder.js +484 -0
  31. package/src/main/agents/index.js +62 -0
  32. package/src/main/agents/orchestrator.js +362 -0
  33. package/src/main/agents/researcher.js +511 -0
  34. package/src/main/agents/state-manager.js +344 -0
  35. package/src/main/agents/supervisor.js +365 -0
  36. package/src/main/agents/verifier.js +452 -0
  37. package/src/main/ai-service.js +1633 -0
  38. package/src/main/index.js +2208 -0
  39. package/src/main/inspect-service.js +467 -0
  40. package/src/main/system-automation.js +1186 -0
  41. package/src/main/ui-automation/config.js +76 -0
  42. package/src/main/ui-automation/core/helpers.js +41 -0
  43. package/src/main/ui-automation/core/index.js +15 -0
  44. package/src/main/ui-automation/core/powershell.js +82 -0
  45. package/src/main/ui-automation/elements/finder.js +274 -0
  46. package/src/main/ui-automation/elements/index.js +14 -0
  47. package/src/main/ui-automation/elements/wait.js +66 -0
  48. package/src/main/ui-automation/index.js +164 -0
  49. package/src/main/ui-automation/interactions/element-click.js +211 -0
  50. package/src/main/ui-automation/interactions/high-level.js +230 -0
  51. package/src/main/ui-automation/interactions/index.js +47 -0
  52. package/src/main/ui-automation/keyboard/index.js +15 -0
  53. package/src/main/ui-automation/keyboard/input.js +179 -0
  54. package/src/main/ui-automation/mouse/click.js +186 -0
  55. package/src/main/ui-automation/mouse/drag.js +88 -0
  56. package/src/main/ui-automation/mouse/index.js +30 -0
  57. package/src/main/ui-automation/mouse/movement.js +51 -0
  58. package/src/main/ui-automation/mouse/scroll.js +116 -0
  59. package/src/main/ui-automation/screenshot.js +183 -0
  60. package/src/main/ui-automation/window/index.js +23 -0
  61. package/src/main/ui-automation/window/manager.js +305 -0
  62. package/src/main/utils/time.js +62 -0
  63. package/src/main/visual-awareness.js +597 -0
  64. package/src/renderer/chat/chat.js +671 -0
  65. package/src/renderer/chat/index.html +725 -0
  66. package/src/renderer/chat/preload.js +112 -0
  67. package/src/renderer/overlay/index.html +648 -0
  68. package/src/renderer/overlay/overlay.js +782 -0
  69. package/src/renderer/overlay/preload.js +90 -0
  70. package/src/shared/grid-math.js +82 -0
  71. package/src/shared/inspect-types.js +230 -0
@@ -0,0 +1,782 @@
1
+ // ===== CONFIGURATION =====
2
+ const gridConfig = window.electronAPI?.getGridConstants
3
+ ? window.electronAPI.getGridConstants()
4
+ : null;
5
+ const COARSE_SPACING = gridConfig?.coarseSpacing || 100; // Coarse grid: 100px spacing
6
+ const FINE_SPACING = gridConfig?.fineSpacing || 25; // Fine grid: 25px spacing
7
+ const START_OFFSET = gridConfig?.startOffset || (COARSE_SPACING / 2); // 50px offset to center grid cells
8
+ const FINE_START = gridConfig?.fineStart || (FINE_SPACING / 2);
9
+ const LOCAL_FINE_RADIUS = gridConfig?.localFineRadius || 3;
10
+
11
+ // ===== STATE MANAGEMENT =====
12
+ let state = {
13
+ currentMode: 'passive',
14
+ zoomLevel: 1, // 1 = coarse, 2 = fine, 3 = all
15
+ width: window.innerWidth,
16
+ height: window.innerHeight,
17
+ mouse: { x: 0, y: 0 },
18
+ indicators: {
19
+ zoom: { visible: false, text: '1x', timeout: null },
20
+ mode: { visible: true, text: 'Selection Mode' },
21
+ feedback: { visible: false, text: '', timeout: null }
22
+ },
23
+ // Inspect mode state
24
+ inspectMode: false,
25
+ inspectRegions: [],
26
+ hoveredRegion: null,
27
+ selectedRegionId: null
28
+ };
29
+
30
+ // ===== CANVAS SETUP =====
31
+ const canvas = document.getElementById('dot-canvas');
32
+ const ctx = canvas.getContext('2d', { alpha: true }); // optimize for alpha
33
+ const container = document.getElementById('overlay-container');
34
+
35
+ // Elements for UI
36
+ const ui = {
37
+ modeIndicator: document.getElementById('mode-indicator'),
38
+ zoomIndicator: document.getElementById('zoom-indicator'),
39
+ statusBar: document.getElementById('status-bar'),
40
+ gridStatus: document.getElementById('grid-status'),
41
+ coordsStatus: document.getElementById('coords-status'),
42
+ interactionRegion: document.getElementById('interaction-region'),
43
+ border: document.getElementById('overlay-border'),
44
+ // Inspect elements
45
+ inspectContainer: document.getElementById('inspect-container'),
46
+ inspectIndicator: document.getElementById('inspect-indicator'),
47
+ inspectTooltip: document.getElementById('inspect-tooltip'),
48
+ regionCount: document.getElementById('region-count')
49
+ };
50
+
51
+ // ===== RENDERING ENGINE =====
52
+ let animationFrameId = null;
53
+ let isDirty = false; // Draw only when needed
54
+
55
+ function requestDraw() {
56
+ if (animationFrameId !== null) return;
57
+ isDirty = true;
58
+ animationFrameId = requestAnimationFrame(draw);
59
+ }
60
+
61
+ function draw() {
62
+ animationFrameId = null;
63
+ if (!isDirty) return;
64
+ isDirty = false;
65
+
66
+ const { width, height, currentMode, zoomLevel } = state;
67
+
68
+ // Clear canvas
69
+ ctx.clearRect(0, 0, width, height);
70
+
71
+ if (currentMode !== 'selection') return;
72
+
73
+ // 1. Draw Coarse Grid (Always visible in selection)
74
+ ctx.fillStyle = 'rgba(0, 122, 255, 0.85)';
75
+ ctx.strokeStyle = 'rgba(255, 255, 255, 0.95)';
76
+ ctx.lineWidth = 2;
77
+
78
+ // Font for labels
79
+ ctx.font = '500 11px "SF Mono", "Monaco", "Menlo", monospace';
80
+ ctx.textAlign = 'center';
81
+ ctx.textBaseline = 'bottom';
82
+
83
+ // Calculate grid bounds
84
+ const cols = Math.ceil((width - START_OFFSET) / COARSE_SPACING) + 1;
85
+ const rows = Math.ceil((height - START_OFFSET) / COARSE_SPACING) + 1;
86
+
87
+ // Draw Coarse Dots + Labels
88
+ for (let c = 0; c < cols; c++) {
89
+ for (let r = 0; r < rows; r++) {
90
+ const x = START_OFFSET + c * COARSE_SPACING;
91
+ const y = START_OFFSET + r * COARSE_SPACING;
92
+
93
+ if (x > width || y > height) continue;
94
+
95
+ // Draw Dot
96
+ ctx.beginPath();
97
+ ctx.arc(x, y, 6, 0, Math.PI * 2);
98
+ ctx.fillStyle = 'rgba(0, 122, 255, 0.85)';
99
+ ctx.fill();
100
+ ctx.stroke();
101
+
102
+ // Draw Label
103
+ const label = generateLabel(c, r, false);
104
+ const metrics = ctx.measureText(label);
105
+ const bgW = metrics.width + 10;
106
+ const bgH = 16;
107
+
108
+ // Label Background
109
+ ctx.fillStyle = 'rgba(0, 0, 0, 0.7)';
110
+ ctx.fillRect(x - bgW / 2, y - 20 - bgH, bgW, bgH);
111
+
112
+ // Label Text
113
+ ctx.fillStyle = 'white';
114
+ ctx.fillText(label, x, y - 24);
115
+ }
116
+ }
117
+
118
+ // 2. Draw Fine Grid (If Zoom Level >= 2)
119
+ if (zoomLevel >= 2) {
120
+ ctx.fillStyle = 'rgba(100, 180, 255, 0.5)';
121
+ ctx.strokeStyle = 'rgba(255, 255, 255, 0.6)';
122
+ ctx.lineWidth = 1;
123
+
124
+ // Performance: Batch all fine dots into one path
125
+ ctx.beginPath();
126
+
127
+ const fCols = Math.ceil(width / FINE_SPACING);
128
+ const fRows = Math.ceil(height / FINE_SPACING);
129
+
130
+ for (let c = 0; c < fCols; c++) {
131
+ for (let r = 0; r < fRows; r++) {
132
+ const x = FINE_START + c * FINE_SPACING;
133
+ const y = FINE_START + r * FINE_SPACING;
134
+
135
+ if (x > width || y > height) continue;
136
+
137
+ // Skip if overlaps with Coarse grid (approx check)
138
+ // Coarse grid is at 50 + n*100.
139
+ const nearestCoarseX = Math.round((x - START_OFFSET)/COARSE_SPACING) * COARSE_SPACING + START_OFFSET;
140
+ const nearestCoarseY = Math.round((y - START_OFFSET)/COARSE_SPACING) * COARSE_SPACING + START_OFFSET;
141
+
142
+ if (Math.abs(x - nearestCoarseX) < 10 && Math.abs(y - nearestCoarseY) < 10) continue;
143
+
144
+ ctx.moveTo(x + 3, y);
145
+ ctx.arc(x, y, 3, 0, Math.PI*2);
146
+ }
147
+ }
148
+ ctx.fill();
149
+ ctx.stroke();
150
+ }
151
+
152
+ // 3. Draw Local Fine Grid (If Zoom Level < 2)
153
+ if (zoomLevel < 2) {
154
+ drawLocalFineGrid();
155
+ }
156
+ }
157
+
158
+ // Resize handler
159
+ function resize() {
160
+ state.width = window.innerWidth;
161
+ state.height = window.innerHeight;
162
+ canvas.width = state.width;
163
+ canvas.height = state.height;
164
+ requestDraw();
165
+ }
166
+ window.addEventListener('resize', resize);
167
+ resize(); // Init
168
+
169
+ // ===== UTILS =====
170
+ function generateLabel(col, row, isFine) {
171
+ if (isFine) {
172
+ // Fine grid logic (B3.21 style)
173
+ const coarseCol = Math.floor(col / 4);
174
+ const coarseRow = Math.floor(row / 4);
175
+ const subCol = col % 4;
176
+ const subRow = row % 4;
177
+ const letter = getColLetter(coarseCol);
178
+ return `${letter}${coarseRow}.${subCol}${subRow}`;
179
+ } else {
180
+ // Coarse grid logic (A1 style)
181
+ const letter = getColLetter(col);
182
+ return `${letter}${row}`;
183
+ }
184
+ }
185
+
186
+ function getColLetter(colIndex) {
187
+ let letter = '';
188
+ if (colIndex >= 26) {
189
+ letter += String.fromCharCode(65 + Math.floor(colIndex / 26) - 1);
190
+ }
191
+ letter += String.fromCharCode(65 + (colIndex % 26));
192
+ return letter;
193
+ }
194
+
195
+ // Coordinate mapping for AI (Inverse of drawing)
196
+ // This must match generateLabel and draw loop logic exactly
197
+ function labelToScreenCoordinates(label) {
198
+ if (window.electronAPI?.labelToScreenCoordinates) {
199
+ return window.electronAPI.labelToScreenCoordinates(label);
200
+ }
201
+ if (!label) return null;
202
+ const match = label.match(/^([A-Z]+)(\d+)(\.(\d)(\d))?$/);
203
+ if (!match) return null;
204
+
205
+ const [, letters, rowStr, , subColStr, subRowStr] = match;
206
+
207
+ // Decode column letters to match getColLetter()
208
+ // A=0..Z=25, AA=26, AB=27, etc.
209
+ let colIndex;
210
+ if (letters.length === 1) {
211
+ colIndex = letters.charCodeAt(0) - 65;
212
+ } else {
213
+ const first = letters.charCodeAt(0) - 65 + 1;
214
+ const second = letters.charCodeAt(1) - 65;
215
+ colIndex = (first * 26) + second;
216
+ }
217
+
218
+ const rowIndex = parseInt(rowStr, 10);
219
+
220
+ if (subColStr && subRowStr) {
221
+ // Fine grid logic: index into the global fine grid (25px spacing)
222
+ const subCol = parseInt(subColStr, 10);
223
+ const subRow = parseInt(subRowStr, 10);
224
+ const fineCol = (colIndex * 4) + subCol;
225
+ const fineRow = (rowIndex * 4) + subRow;
226
+ const fineX = FINE_START + fineCol * FINE_SPACING;
227
+ const fineY = FINE_START + fineRow * FINE_SPACING;
228
+ return { x: fineX, y: fineY, screenX: fineX, screenY: fineY };
229
+ } else {
230
+ // Coarse
231
+ const x = START_OFFSET + colIndex * COARSE_SPACING;
232
+ const y = START_OFFSET + rowIndex * COARSE_SPACING;
233
+ return { x, y, screenX: x, screenY: y };
234
+ }
235
+ }
236
+
237
+ function drawLocalFineGrid() {
238
+ if (state.currentMode !== 'selection') return;
239
+ const { mouse, width, height } = state;
240
+ if (!mouse) return;
241
+
242
+ const baseCol = Math.round((mouse.x - FINE_START) / FINE_SPACING);
243
+ const baseRow = Math.round((mouse.y - FINE_START) / FINE_SPACING);
244
+
245
+ const minCol = baseCol - LOCAL_FINE_RADIUS;
246
+ const maxCol = baseCol + LOCAL_FINE_RADIUS;
247
+ const minRow = baseRow - LOCAL_FINE_RADIUS;
248
+ const maxRow = baseRow + LOCAL_FINE_RADIUS;
249
+
250
+ ctx.fillStyle = 'rgba(120, 200, 255, 0.7)';
251
+ ctx.strokeStyle = 'rgba(255, 255, 255, 0.75)';
252
+ ctx.lineWidth = 1;
253
+
254
+ ctx.beginPath();
255
+ for (let c = minCol; c <= maxCol; c++) {
256
+ const x = FINE_START + c * FINE_SPACING;
257
+ if (x < 0 || x > width) continue;
258
+ for (let r = minRow; r <= maxRow; r++) {
259
+ const y = FINE_START + r * FINE_SPACING;
260
+ if (y < 0 || y > height) continue;
261
+ ctx.moveTo(x + 2, y);
262
+ ctx.arc(x, y, 2, 0, Math.PI * 2);
263
+ }
264
+ }
265
+ ctx.fill();
266
+ ctx.stroke();
267
+
268
+ const centerX = FINE_START + baseCol * FINE_SPACING;
269
+ const centerY = FINE_START + baseRow * FINE_SPACING;
270
+ if (centerX >= 0 && centerX <= width && centerY >= 0 && centerY <= height) {
271
+ ctx.beginPath();
272
+ ctx.arc(centerX, centerY, 4, 0, Math.PI * 2);
273
+ ctx.strokeStyle = 'rgba(0, 255, 200, 0.9)';
274
+ ctx.lineWidth = 2;
275
+ ctx.stroke();
276
+ }
277
+ }
278
+
279
+ // ===== INPUT HANDLING =====
280
+
281
+ // Visual Feedback Helper
282
+ function showFeedback(text) {
283
+ const el = document.getElementById('key-feedback');
284
+ let fb = el;
285
+ if(!fb) {
286
+ fb = document.createElement('div');
287
+ fb.id = 'key-feedback';
288
+ fb.style.cssText = `position:fixed; top:50%; left:50%; transform:translate(-50%,-50%);
289
+ background:rgba(0,120,215,0.9); color:white; padding:16px 32px; border-radius:8px;
290
+ font-size:18px; font-weight:600; opacity:0; transition:opacity 0.2s; pointer-events:none; z-index:99999;`;
291
+ document.body.appendChild(fb);
292
+ }
293
+ fb.textContent = text;
294
+ fb.style.opacity = 1;
295
+ clearTimeout(state.indicators.feedback.timeout);
296
+ state.indicators.feedback.timeout = setTimeout(() => fb.style.opacity = 0, 1000);
297
+ }
298
+
299
+ // Mouse Tracking for Virtual Interaction
300
+ document.addEventListener('mousemove', (e) => {
301
+ state.mouse = { x: e.clientX, y: e.clientY };
302
+ if(ui.coordsStatus) ui.coordsStatus.textContent = `${e.clientX}, ${e.clientY}`;
303
+
304
+ if (state.currentMode === 'selection') {
305
+ requestDraw();
306
+ // Virtual Interaction Logic
307
+ // Find nearest grid point
308
+ const spacing = state.zoomLevel >= 2 ? FINE_SPACING : COARSE_SPACING;
309
+ const offset = state.zoomLevel >= 2 ? FINE_START : START_OFFSET;
310
+
311
+ // Nearest index
312
+ const c = Math.round((e.clientX - offset) / spacing);
313
+ const r = Math.round((e.clientY - offset) / spacing);
314
+ const snapX = offset + c * spacing;
315
+ const snapY = offset + r * spacing;
316
+
317
+ // Dist
318
+ const dx = e.clientX - snapX;
319
+ const dy = e.clientY - snapY;
320
+ const dist = Math.sqrt(dx*dx + dy*dy);
321
+
322
+ // Highlight if close
323
+ if (dist < 30) {
324
+ if(ui.interactionRegion) {
325
+ ui.interactionRegion.style.left = (snapX - 15) + 'px';
326
+ ui.interactionRegion.style.top = (snapY - 15) + 'px';
327
+ ui.interactionRegion.style.width = '30px';
328
+ ui.interactionRegion.style.height = '30px';
329
+ ui.interactionRegion.classList.add('visible');
330
+ ui.interactionRegion.dataset.x = snapX;
331
+ ui.interactionRegion.dataset.y = snapY;
332
+ }
333
+ } else {
334
+ if(ui.interactionRegion) ui.interactionRegion.classList.remove('visible');
335
+ }
336
+ }
337
+ });
338
+
339
+ document.addEventListener('click', (e) => {
340
+ if (state.currentMode === 'selection' && ui.interactionRegion && ui.interactionRegion.classList.contains('visible')) {
341
+ const x = parseFloat(ui.interactionRegion.dataset.x);
342
+ const y = parseFloat(ui.interactionRegion.dataset.y);
343
+
344
+ // Flash effect
345
+ showPulse(x, y);
346
+
347
+ // Send to main
348
+ let label;
349
+ let type;
350
+ if (state.zoomLevel >= 2) {
351
+ const fineCol = Math.round((x - FINE_START) / FINE_SPACING);
352
+ const fineRow = Math.round((y - FINE_START) / FINE_SPACING);
353
+ label = generateLabel(fineCol, fineRow, true);
354
+ type = 'fine';
355
+ } else {
356
+ const colInit = Math.round((x - START_OFFSET) / COARSE_SPACING);
357
+ const rowInit = Math.round((y - START_OFFSET) / COARSE_SPACING);
358
+ label = generateLabel(colInit, rowInit, false);
359
+ type = 'coarse';
360
+ }
361
+
362
+ if(window.electronAPI) {
363
+ window.electronAPI.selectDot({
364
+ id: `virtual-${x}-${y}`,
365
+ x, y, bg: true, label,
366
+ screenX: x, screenY: y,
367
+ type
368
+ });
369
+ }
370
+ }
371
+ });
372
+
373
+ // Pulse Effect (Doppler)
374
+ function showPulse(x, y) {
375
+ const el = document.createElement('div');
376
+ el.className = 'pulse-ring';
377
+ el.style.cssText = `position:fixed; left:${x}px; top:${y}px; width:10px; height:10px;
378
+ transform:translate(-50%,-50%); background:rgba(0,255,200,0.5); border-radius:50%;
379
+ box-shadow: 0 0 15px rgba(0,255,200,0.8); border: 2px solid #00ffcc;
380
+ transition:all 0.6s cubic-bezier(0.25, 0.46, 0.45, 0.94); pointer-events:none; z-index:2147483647;`;
381
+ document.body.appendChild(el);
382
+ requestAnimationFrame(() => {
383
+ el.style.width = '120px';
384
+ el.style.height = '120px';
385
+ el.style.opacity = 0;
386
+ el.style.borderWidth = '0px';
387
+ });
388
+ setTimeout(() => el.remove(), 700);
389
+ }
390
+
391
+ // ===== IPC & COMMANDS =====
392
+ if (window.electronAPI) {
393
+ window.electronAPI.onModeChanged((mode) => {
394
+ state.currentMode = mode;
395
+ state.zoomLevel = 1;
396
+
397
+ if (mode === 'selection') {
398
+ if(ui.modeIndicator) ui.modeIndicator.classList.add('visible');
399
+ if(ui.border) ui.border.classList.add('active');
400
+ } else {
401
+ if(ui.modeIndicator) ui.modeIndicator.classList.remove('visible');
402
+ if(ui.border) ui.border.classList.remove('active');
403
+ if(ui.interactionRegion) ui.interactionRegion.classList.remove('visible');
404
+ }
405
+ requestDraw();
406
+ });
407
+
408
+ window.electronAPI.onOverlayCommand((data) => {
409
+ handleCommand(data);
410
+ });
411
+
412
+ // Initialize State from Main Process
413
+ window.electronAPI.getState().then(initialState => {
414
+ console.log('Initial state loaded:', initialState);
415
+ if (initialState.overlayMode) {
416
+ state.currentMode = initialState.overlayMode;
417
+ // If valid mode, trigger UI update
418
+ if (state.currentMode === 'selection') {
419
+ if(ui.modeIndicator) ui.modeIndicator.classList.add('visible');
420
+ if(ui.border) ui.border.classList.add('active');
421
+ }
422
+ requestDraw();
423
+ }
424
+ // Load inspect mode state if available
425
+ if (initialState.inspectMode !== undefined) {
426
+ state.inspectMode = initialState.inspectMode;
427
+ updateInspectIndicator();
428
+ }
429
+ }).catch(err => console.error('Failed to get initial state:', err));
430
+
431
+ // Listen for inspect regions update
432
+ if (window.electronAPI.onInspectRegionsUpdate) {
433
+ window.electronAPI.onInspectRegionsUpdate((regions) => {
434
+ console.log('Received inspect regions:', regions?.length || 0);
435
+ updateInspectRegions(regions);
436
+ });
437
+ }
438
+
439
+ // Listen for inspect mode toggle
440
+ if (window.electronAPI.onInspectModeChanged) {
441
+ window.electronAPI.onInspectModeChanged((enabled) => {
442
+ console.log('Inspect mode changed:', enabled);
443
+ state.inspectMode = enabled;
444
+ updateInspectIndicator();
445
+ if (!enabled) {
446
+ clearInspectRegions();
447
+ }
448
+ });
449
+ }
450
+
451
+ // Identify
452
+ console.log('Hooked electronAPI events');
453
+ } else {
454
+ console.warn('electronAPI not found - running in standalone mode?');
455
+ }
456
+
457
+ function handleCommand(data) {
458
+ console.log('Command:', data.action);
459
+ switch (data.action) {
460
+ case 'toggle-fine':
461
+ state.zoomLevel = state.zoomLevel >= 2 ? 1 : 2;
462
+ showFeedback(state.zoomLevel >= 2 ? 'Fine Grid ON' : 'Fine Grid OFF');
463
+ requestDraw();
464
+ break;
465
+ case 'show-all':
466
+ state.zoomLevel = 3;
467
+ showFeedback('All Grids Visible');
468
+ requestDraw();
469
+ break;
470
+ case 'zoom-in':
471
+ state.zoomLevel = Math.min(3, state.zoomLevel + 1);
472
+ showFeedback(`Zoom: ${state.zoomLevel}x`);
473
+ requestDraw();
474
+ break;
475
+ case 'zoom-out':
476
+ state.zoomLevel = Math.max(1, state.zoomLevel - 1);
477
+ showFeedback(`Zoom: ${state.zoomLevel}x`);
478
+ requestDraw();
479
+ break;
480
+ case 'set-click-through':
481
+ document.body.style.pointerEvents = data.enabled ? 'none' : '';
482
+ if(ui.interactionRegion) ui.interactionRegion.style.pointerEvents = data.enabled ? 'none' : '';
483
+ // Also update inspect regions pointer events
484
+ if(ui.inspectContainer) ui.inspectContainer.style.pointerEvents = data.enabled ? 'none' : '';
485
+ break;
486
+ case 'pulse-click':
487
+ case 'highlight-coordinate':
488
+ showPulse(data.x, data.y);
489
+ break;
490
+ case 'get-coordinates':
491
+ if (data.label && window.electronAPI.sendCoordinates) {
492
+ // Not implemented in preload yet, but logical place
493
+ // For now, we rely on main process calculating it via ai-service
494
+ }
495
+ break;
496
+ // Inspect mode commands
497
+ case 'toggle-inspect':
498
+ state.inspectMode = !state.inspectMode;
499
+ showFeedback(state.inspectMode ? 'Inspect Mode ON' : 'Inspect Mode OFF');
500
+ updateInspectIndicator();
501
+ if (!state.inspectMode) {
502
+ clearInspectRegions();
503
+ }
504
+ break;
505
+ case 'update-inspect-regions':
506
+ if (data.regions) {
507
+ updateInspectRegions(data.regions);
508
+ }
509
+ break;
510
+ case 'clear-inspect-regions':
511
+ clearInspectRegions();
512
+ break;
513
+ }
514
+
515
+ if (ui.gridStatus) {
516
+ ui.gridStatus.textContent = state.zoomLevel > 1 ? 'Grid: Fine' : 'Grid: Coarse';
517
+ }
518
+ }
519
+
520
+ // ===== INSPECT MODE FUNCTIONS =====
521
+
522
+ /**
523
+ * Update inspect indicator visibility
524
+ */
525
+ function updateInspectIndicator() {
526
+ if (ui.inspectIndicator) {
527
+ if (state.inspectMode) {
528
+ ui.inspectIndicator.classList.add('visible');
529
+ } else {
530
+ ui.inspectIndicator.classList.remove('visible');
531
+ }
532
+ }
533
+ }
534
+
535
+ /**
536
+ * Update inspect regions display
537
+ * @param {Array} regions - Array of region objects with bounds, label, role, confidence
538
+ */
539
+ function updateInspectRegions(regions) {
540
+ if (!ui.inspectContainer) return;
541
+
542
+ // Clear existing regions
543
+ ui.inspectContainer.innerHTML = '';
544
+ state.inspectRegions = regions || [];
545
+
546
+ // Update region count
547
+ if (ui.regionCount) {
548
+ ui.regionCount.textContent = state.inspectRegions.length;
549
+ }
550
+
551
+ // Render regions
552
+ state.inspectRegions.forEach((region, index) => {
553
+ const el = createRegionElement(region, index);
554
+ ui.inspectContainer.appendChild(el);
555
+ });
556
+
557
+ console.log(`Rendered ${state.inspectRegions.length} inspect regions`);
558
+ }
559
+
560
+ /**
561
+ * Create a DOM element for an inspect region
562
+ * @param {Object} region - Region data
563
+ * @param {number} index - Region index
564
+ * @returns {HTMLElement}
565
+ */
566
+ function createRegionElement(region, index) {
567
+ const el = document.createElement('div');
568
+ el.className = 'inspect-region';
569
+ el.dataset.regionId = region.id;
570
+ el.dataset.index = index;
571
+
572
+ // Position and size
573
+ const bounds = region.bounds || {};
574
+ el.style.left = `${bounds.x || 0}px`;
575
+ el.style.top = `${bounds.y || 0}px`;
576
+ el.style.width = `${bounds.width || 0}px`;
577
+ el.style.height = `${bounds.height || 0}px`;
578
+
579
+ // Add classes for state
580
+ // Handle undefined/null confidence - default to 1.0 (high confidence)
581
+ const confidence = region.confidence ?? 1.0;
582
+ if (confidence < 0.7) {
583
+ el.classList.add('low-confidence');
584
+ }
585
+ if (region.id === state.selectedRegionId) {
586
+ el.classList.add('selected');
587
+ }
588
+
589
+ // Add label
590
+ const label = document.createElement('span');
591
+ label.className = 'inspect-region-label';
592
+ label.textContent = region.label || region.role || `Region ${index + 1}`;
593
+ el.appendChild(label);
594
+
595
+ // Event handlers
596
+ el.addEventListener('mouseenter', (e) => {
597
+ state.hoveredRegion = region;
598
+ showInspectTooltip(region, e.clientX, e.clientY);
599
+ });
600
+
601
+ el.addEventListener('mouseleave', () => {
602
+ state.hoveredRegion = null;
603
+ hideInspectTooltip();
604
+ });
605
+
606
+ el.addEventListener('mousemove', (e) => {
607
+ if (state.hoveredRegion === region) {
608
+ positionTooltip(e.clientX, e.clientY);
609
+ }
610
+ });
611
+
612
+ el.addEventListener('click', (e) => {
613
+ e.stopPropagation();
614
+ selectRegion(region);
615
+ });
616
+
617
+ return el;
618
+ }
619
+
620
+ /**
621
+ * Show inspect tooltip for a region
622
+ * @param {Object} region - Region data
623
+ * @param {number} x - Mouse X position
624
+ * @param {number} y - Mouse Y position
625
+ */
626
+ function showInspectTooltip(region, x, y) {
627
+ if (!ui.inspectTooltip) return;
628
+
629
+ // Update tooltip content
630
+ const roleEl = ui.inspectTooltip.querySelector('.tooltip-role');
631
+ const labelEl = ui.inspectTooltip.querySelector('.tooltip-label');
632
+ const textEl = document.getElementById('tooltip-text');
633
+ const posEl = document.getElementById('tooltip-position');
634
+ const confEl = document.getElementById('tooltip-confidence');
635
+ const confBar = document.getElementById('tooltip-confidence-bar');
636
+
637
+ if (roleEl) roleEl.textContent = region.role || 'element';
638
+ if (labelEl) labelEl.textContent = region.label || 'Unknown';
639
+ if (textEl) textEl.textContent = region.text || '-';
640
+
641
+ const centerX = Math.round((region.bounds?.x || 0) + (region.bounds?.width || 0) / 2);
642
+ const centerY = Math.round((region.bounds?.y || 0) + (region.bounds?.height || 0) / 2);
643
+ if (posEl) posEl.textContent = `${centerX}, ${centerY}`;
644
+
645
+ const confidence = Math.round((region.confidence || 0.5) * 100);
646
+ if (confEl) confEl.textContent = `${confidence}%`;
647
+ if (confBar) confBar.style.width = `${confidence}%`;
648
+
649
+ // Position and show tooltip
650
+ positionTooltip(x, y);
651
+ ui.inspectTooltip.classList.add('visible');
652
+ }
653
+
654
+ /**
655
+ * Position tooltip near cursor
656
+ * @param {number} x - Mouse X
657
+ * @param {number} y - Mouse Y
658
+ */
659
+ function positionTooltip(x, y) {
660
+ if (!ui.inspectTooltip) return;
661
+
662
+ const offset = 15;
663
+ const tooltipRect = ui.inspectTooltip.getBoundingClientRect();
664
+
665
+ // Default position: below and to the right of cursor
666
+ let left = x + offset;
667
+ let top = y + offset;
668
+
669
+ // Adjust if tooltip would go off screen
670
+ if (left + tooltipRect.width > window.innerWidth) {
671
+ left = x - tooltipRect.width - offset;
672
+ }
673
+ if (top + tooltipRect.height > window.innerHeight) {
674
+ top = y - tooltipRect.height - offset;
675
+ }
676
+
677
+ ui.inspectTooltip.style.left = `${left}px`;
678
+ ui.inspectTooltip.style.top = `${top}px`;
679
+ }
680
+
681
+ /**
682
+ * Hide inspect tooltip
683
+ */
684
+ function hideInspectTooltip() {
685
+ if (ui.inspectTooltip) {
686
+ ui.inspectTooltip.classList.remove('visible');
687
+ }
688
+ }
689
+
690
+ /**
691
+ * Select a region and notify main process
692
+ * @param {Object} region - Region to select
693
+ */
694
+ function selectRegion(region) {
695
+ // Update state
696
+ state.selectedRegionId = region.id;
697
+
698
+ // Update visual state
699
+ document.querySelectorAll('.inspect-region').forEach(el => {
700
+ el.classList.remove('selected');
701
+ if (el.dataset.regionId === region.id) {
702
+ el.classList.add('selected');
703
+ }
704
+ });
705
+
706
+ // Show pulse at region center
707
+ const centerX = (region.bounds?.x || 0) + (region.bounds?.width || 0) / 2;
708
+ const centerY = (region.bounds?.y || 0) + (region.bounds?.height || 0) / 2;
709
+ showPulse(centerX, centerY);
710
+
711
+ // Notify main process
712
+ if (window.electronAPI?.selectInspectRegion) {
713
+ window.electronAPI.selectInspectRegion({
714
+ targetId: region.id,
715
+ region: region,
716
+ bounds: region.bounds,
717
+ x: centerX,
718
+ y: centerY
719
+ });
720
+ } else if (window.electronAPI?.selectDot) {
721
+ // Fallback to dot selection
722
+ window.electronAPI.selectDot({
723
+ id: `inspect-${region.id}`,
724
+ x: centerX,
725
+ y: centerY,
726
+ label: region.label || region.role,
727
+ targetId: region.id,
728
+ type: 'inspect-region',
729
+ screenX: centerX,
730
+ screenY: centerY,
731
+ region: region
732
+ });
733
+ }
734
+
735
+ showFeedback(`Selected: ${region.label || region.role || 'Region'}`);
736
+ }
737
+
738
+ /**
739
+ * Clear all inspect regions
740
+ */
741
+ function clearInspectRegions() {
742
+ if (ui.inspectContainer) {
743
+ ui.inspectContainer.innerHTML = '';
744
+ }
745
+ state.inspectRegions = [];
746
+ state.hoveredRegion = null;
747
+ state.selectedRegionId = null;
748
+
749
+ if (ui.regionCount) {
750
+ ui.regionCount.textContent = '0';
751
+ }
752
+
753
+ hideInspectTooltip();
754
+ }
755
+
756
+ /**
757
+ * Find region at a point (for hover detection)
758
+ * Uses exclusive bounds (x < right, y < bottom) for correct hit detection
759
+ * @param {number} x - X coordinate
760
+ * @param {number} y - Y coordinate
761
+ * @returns {Object|null}
762
+ */
763
+ function findRegionAtPoint(x, y) {
764
+ for (const region of state.inspectRegions) {
765
+ const b = region.bounds;
766
+ // Use exclusive bounds (< instead of <=) for mathematical correctness
767
+ if (x >= b.x && x < b.x + b.width && y >= b.y && y < b.y + b.height) {
768
+ return region;
769
+ }
770
+ }
771
+ return null;
772
+ }
773
+
774
+ // Expose Helper Global
775
+ window.labelToScreenCoordinates = labelToScreenCoordinates;
776
+
777
+ // Expose inspect functions globally for debugging
778
+ window.updateInspectRegions = updateInspectRegions;
779
+ window.clearInspectRegions = clearInspectRegions;
780
+
781
+ console.log('High-Performance Canvas Overlay Loaded');
782
+ requestDraw();