copilot-liku-cli 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +411 -0
- package/CONFIGURATION.md +302 -0
- package/CONTRIBUTING.md +225 -0
- package/ELECTRON_README.md +121 -0
- package/INSTALLATION.md +350 -0
- package/LICENSE.md +1 -0
- package/PROJECT_STATUS.md +229 -0
- package/QUICKSTART.md +255 -0
- package/README.md +167 -0
- package/TESTING.md +274 -0
- package/package.json +61 -0
- package/scripts/start.js +30 -0
- package/src/assets/tray-icon.png +0 -0
- package/src/cli/commands/agent.js +327 -0
- package/src/cli/commands/click.js +108 -0
- package/src/cli/commands/drag.js +85 -0
- package/src/cli/commands/find.js +109 -0
- package/src/cli/commands/keys.js +132 -0
- package/src/cli/commands/mouse.js +79 -0
- package/src/cli/commands/repl.js +290 -0
- package/src/cli/commands/screenshot.js +72 -0
- package/src/cli/commands/scroll.js +74 -0
- package/src/cli/commands/start.js +67 -0
- package/src/cli/commands/type.js +57 -0
- package/src/cli/commands/wait.js +84 -0
- package/src/cli/commands/window.js +104 -0
- package/src/cli/liku.js +249 -0
- package/src/cli/util/output.js +174 -0
- package/src/main/agents/base-agent.js +410 -0
- package/src/main/agents/builder.js +484 -0
- package/src/main/agents/index.js +62 -0
- package/src/main/agents/orchestrator.js +362 -0
- package/src/main/agents/researcher.js +511 -0
- package/src/main/agents/state-manager.js +344 -0
- package/src/main/agents/supervisor.js +365 -0
- package/src/main/agents/verifier.js +452 -0
- package/src/main/ai-service.js +1633 -0
- package/src/main/index.js +2208 -0
- package/src/main/inspect-service.js +467 -0
- package/src/main/system-automation.js +1186 -0
- package/src/main/ui-automation/config.js +76 -0
- package/src/main/ui-automation/core/helpers.js +41 -0
- package/src/main/ui-automation/core/index.js +15 -0
- package/src/main/ui-automation/core/powershell.js +82 -0
- package/src/main/ui-automation/elements/finder.js +274 -0
- package/src/main/ui-automation/elements/index.js +14 -0
- package/src/main/ui-automation/elements/wait.js +66 -0
- package/src/main/ui-automation/index.js +164 -0
- package/src/main/ui-automation/interactions/element-click.js +211 -0
- package/src/main/ui-automation/interactions/high-level.js +230 -0
- package/src/main/ui-automation/interactions/index.js +47 -0
- package/src/main/ui-automation/keyboard/index.js +15 -0
- package/src/main/ui-automation/keyboard/input.js +179 -0
- package/src/main/ui-automation/mouse/click.js +186 -0
- package/src/main/ui-automation/mouse/drag.js +88 -0
- package/src/main/ui-automation/mouse/index.js +30 -0
- package/src/main/ui-automation/mouse/movement.js +51 -0
- package/src/main/ui-automation/mouse/scroll.js +116 -0
- package/src/main/ui-automation/screenshot.js +183 -0
- package/src/main/ui-automation/window/index.js +23 -0
- package/src/main/ui-automation/window/manager.js +305 -0
- package/src/main/utils/time.js +62 -0
- package/src/main/visual-awareness.js +597 -0
- package/src/renderer/chat/chat.js +671 -0
- package/src/renderer/chat/index.html +725 -0
- package/src/renderer/chat/preload.js +112 -0
- package/src/renderer/overlay/index.html +648 -0
- package/src/renderer/overlay/overlay.js +782 -0
- package/src/renderer/overlay/preload.js +90 -0
- package/src/shared/grid-math.js +82 -0
- package/src/shared/inspect-types.js +230 -0
|
@@ -0,0 +1,782 @@
|
|
|
1
|
+
// ===== CONFIGURATION =====
|
|
2
|
+
const gridConfig = window.electronAPI?.getGridConstants
|
|
3
|
+
? window.electronAPI.getGridConstants()
|
|
4
|
+
: null;
|
|
5
|
+
const COARSE_SPACING = gridConfig?.coarseSpacing || 100; // Coarse grid: 100px spacing
|
|
6
|
+
const FINE_SPACING = gridConfig?.fineSpacing || 25; // Fine grid: 25px spacing
|
|
7
|
+
const START_OFFSET = gridConfig?.startOffset || (COARSE_SPACING / 2); // 50px offset to center grid cells
|
|
8
|
+
const FINE_START = gridConfig?.fineStart || (FINE_SPACING / 2);
|
|
9
|
+
const LOCAL_FINE_RADIUS = gridConfig?.localFineRadius || 3;
|
|
10
|
+
|
|
11
|
+
// ===== STATE MANAGEMENT =====
|
|
12
|
+
let state = {
|
|
13
|
+
currentMode: 'passive',
|
|
14
|
+
zoomLevel: 1, // 1 = coarse, 2 = fine, 3 = all
|
|
15
|
+
width: window.innerWidth,
|
|
16
|
+
height: window.innerHeight,
|
|
17
|
+
mouse: { x: 0, y: 0 },
|
|
18
|
+
indicators: {
|
|
19
|
+
zoom: { visible: false, text: '1x', timeout: null },
|
|
20
|
+
mode: { visible: true, text: 'Selection Mode' },
|
|
21
|
+
feedback: { visible: false, text: '', timeout: null }
|
|
22
|
+
},
|
|
23
|
+
// Inspect mode state
|
|
24
|
+
inspectMode: false,
|
|
25
|
+
inspectRegions: [],
|
|
26
|
+
hoveredRegion: null,
|
|
27
|
+
selectedRegionId: null
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
// ===== CANVAS SETUP =====
|
|
31
|
+
const canvas = document.getElementById('dot-canvas');
|
|
32
|
+
const ctx = canvas.getContext('2d', { alpha: true }); // optimize for alpha
|
|
33
|
+
const container = document.getElementById('overlay-container');
|
|
34
|
+
|
|
35
|
+
// Elements for UI
|
|
36
|
+
const ui = {
|
|
37
|
+
modeIndicator: document.getElementById('mode-indicator'),
|
|
38
|
+
zoomIndicator: document.getElementById('zoom-indicator'),
|
|
39
|
+
statusBar: document.getElementById('status-bar'),
|
|
40
|
+
gridStatus: document.getElementById('grid-status'),
|
|
41
|
+
coordsStatus: document.getElementById('coords-status'),
|
|
42
|
+
interactionRegion: document.getElementById('interaction-region'),
|
|
43
|
+
border: document.getElementById('overlay-border'),
|
|
44
|
+
// Inspect elements
|
|
45
|
+
inspectContainer: document.getElementById('inspect-container'),
|
|
46
|
+
inspectIndicator: document.getElementById('inspect-indicator'),
|
|
47
|
+
inspectTooltip: document.getElementById('inspect-tooltip'),
|
|
48
|
+
regionCount: document.getElementById('region-count')
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
// ===== RENDERING ENGINE =====
|
|
52
|
+
let animationFrameId = null;
|
|
53
|
+
let isDirty = false; // Draw only when needed
|
|
54
|
+
|
|
55
|
+
function requestDraw() {
|
|
56
|
+
if (animationFrameId !== null) return;
|
|
57
|
+
isDirty = true;
|
|
58
|
+
animationFrameId = requestAnimationFrame(draw);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function draw() {
|
|
62
|
+
animationFrameId = null;
|
|
63
|
+
if (!isDirty) return;
|
|
64
|
+
isDirty = false;
|
|
65
|
+
|
|
66
|
+
const { width, height, currentMode, zoomLevel } = state;
|
|
67
|
+
|
|
68
|
+
// Clear canvas
|
|
69
|
+
ctx.clearRect(0, 0, width, height);
|
|
70
|
+
|
|
71
|
+
if (currentMode !== 'selection') return;
|
|
72
|
+
|
|
73
|
+
// 1. Draw Coarse Grid (Always visible in selection)
|
|
74
|
+
ctx.fillStyle = 'rgba(0, 122, 255, 0.85)';
|
|
75
|
+
ctx.strokeStyle = 'rgba(255, 255, 255, 0.95)';
|
|
76
|
+
ctx.lineWidth = 2;
|
|
77
|
+
|
|
78
|
+
// Font for labels
|
|
79
|
+
ctx.font = '500 11px "SF Mono", "Monaco", "Menlo", monospace';
|
|
80
|
+
ctx.textAlign = 'center';
|
|
81
|
+
ctx.textBaseline = 'bottom';
|
|
82
|
+
|
|
83
|
+
// Calculate grid bounds
|
|
84
|
+
const cols = Math.ceil((width - START_OFFSET) / COARSE_SPACING) + 1;
|
|
85
|
+
const rows = Math.ceil((height - START_OFFSET) / COARSE_SPACING) + 1;
|
|
86
|
+
|
|
87
|
+
// Draw Coarse Dots + Labels
|
|
88
|
+
for (let c = 0; c < cols; c++) {
|
|
89
|
+
for (let r = 0; r < rows; r++) {
|
|
90
|
+
const x = START_OFFSET + c * COARSE_SPACING;
|
|
91
|
+
const y = START_OFFSET + r * COARSE_SPACING;
|
|
92
|
+
|
|
93
|
+
if (x > width || y > height) continue;
|
|
94
|
+
|
|
95
|
+
// Draw Dot
|
|
96
|
+
ctx.beginPath();
|
|
97
|
+
ctx.arc(x, y, 6, 0, Math.PI * 2);
|
|
98
|
+
ctx.fillStyle = 'rgba(0, 122, 255, 0.85)';
|
|
99
|
+
ctx.fill();
|
|
100
|
+
ctx.stroke();
|
|
101
|
+
|
|
102
|
+
// Draw Label
|
|
103
|
+
const label = generateLabel(c, r, false);
|
|
104
|
+
const metrics = ctx.measureText(label);
|
|
105
|
+
const bgW = metrics.width + 10;
|
|
106
|
+
const bgH = 16;
|
|
107
|
+
|
|
108
|
+
// Label Background
|
|
109
|
+
ctx.fillStyle = 'rgba(0, 0, 0, 0.7)';
|
|
110
|
+
ctx.fillRect(x - bgW / 2, y - 20 - bgH, bgW, bgH);
|
|
111
|
+
|
|
112
|
+
// Label Text
|
|
113
|
+
ctx.fillStyle = 'white';
|
|
114
|
+
ctx.fillText(label, x, y - 24);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// 2. Draw Fine Grid (If Zoom Level >= 2)
|
|
119
|
+
if (zoomLevel >= 2) {
|
|
120
|
+
ctx.fillStyle = 'rgba(100, 180, 255, 0.5)';
|
|
121
|
+
ctx.strokeStyle = 'rgba(255, 255, 255, 0.6)';
|
|
122
|
+
ctx.lineWidth = 1;
|
|
123
|
+
|
|
124
|
+
// Performance: Batch all fine dots into one path
|
|
125
|
+
ctx.beginPath();
|
|
126
|
+
|
|
127
|
+
const fCols = Math.ceil(width / FINE_SPACING);
|
|
128
|
+
const fRows = Math.ceil(height / FINE_SPACING);
|
|
129
|
+
|
|
130
|
+
for (let c = 0; c < fCols; c++) {
|
|
131
|
+
for (let r = 0; r < fRows; r++) {
|
|
132
|
+
const x = FINE_START + c * FINE_SPACING;
|
|
133
|
+
const y = FINE_START + r * FINE_SPACING;
|
|
134
|
+
|
|
135
|
+
if (x > width || y > height) continue;
|
|
136
|
+
|
|
137
|
+
// Skip if overlaps with Coarse grid (approx check)
|
|
138
|
+
// Coarse grid is at 50 + n*100.
|
|
139
|
+
const nearestCoarseX = Math.round((x - START_OFFSET)/COARSE_SPACING) * COARSE_SPACING + START_OFFSET;
|
|
140
|
+
const nearestCoarseY = Math.round((y - START_OFFSET)/COARSE_SPACING) * COARSE_SPACING + START_OFFSET;
|
|
141
|
+
|
|
142
|
+
if (Math.abs(x - nearestCoarseX) < 10 && Math.abs(y - nearestCoarseY) < 10) continue;
|
|
143
|
+
|
|
144
|
+
ctx.moveTo(x + 3, y);
|
|
145
|
+
ctx.arc(x, y, 3, 0, Math.PI*2);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
ctx.fill();
|
|
149
|
+
ctx.stroke();
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// 3. Draw Local Fine Grid (If Zoom Level < 2)
|
|
153
|
+
if (zoomLevel < 2) {
|
|
154
|
+
drawLocalFineGrid();
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Resize handler
|
|
159
|
+
function resize() {
|
|
160
|
+
state.width = window.innerWidth;
|
|
161
|
+
state.height = window.innerHeight;
|
|
162
|
+
canvas.width = state.width;
|
|
163
|
+
canvas.height = state.height;
|
|
164
|
+
requestDraw();
|
|
165
|
+
}
|
|
166
|
+
window.addEventListener('resize', resize);
|
|
167
|
+
resize(); // Init
|
|
168
|
+
|
|
169
|
+
// ===== UTILS =====
|
|
170
|
+
function generateLabel(col, row, isFine) {
|
|
171
|
+
if (isFine) {
|
|
172
|
+
// Fine grid logic (B3.21 style)
|
|
173
|
+
const coarseCol = Math.floor(col / 4);
|
|
174
|
+
const coarseRow = Math.floor(row / 4);
|
|
175
|
+
const subCol = col % 4;
|
|
176
|
+
const subRow = row % 4;
|
|
177
|
+
const letter = getColLetter(coarseCol);
|
|
178
|
+
return `${letter}${coarseRow}.${subCol}${subRow}`;
|
|
179
|
+
} else {
|
|
180
|
+
// Coarse grid logic (A1 style)
|
|
181
|
+
const letter = getColLetter(col);
|
|
182
|
+
return `${letter}${row}`;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function getColLetter(colIndex) {
|
|
187
|
+
let letter = '';
|
|
188
|
+
if (colIndex >= 26) {
|
|
189
|
+
letter += String.fromCharCode(65 + Math.floor(colIndex / 26) - 1);
|
|
190
|
+
}
|
|
191
|
+
letter += String.fromCharCode(65 + (colIndex % 26));
|
|
192
|
+
return letter;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// Coordinate mapping for AI (Inverse of drawing)
|
|
196
|
+
// This must match generateLabel and draw loop logic exactly
|
|
197
|
+
function labelToScreenCoordinates(label) {
|
|
198
|
+
if (window.electronAPI?.labelToScreenCoordinates) {
|
|
199
|
+
return window.electronAPI.labelToScreenCoordinates(label);
|
|
200
|
+
}
|
|
201
|
+
if (!label) return null;
|
|
202
|
+
const match = label.match(/^([A-Z]+)(\d+)(\.(\d)(\d))?$/);
|
|
203
|
+
if (!match) return null;
|
|
204
|
+
|
|
205
|
+
const [, letters, rowStr, , subColStr, subRowStr] = match;
|
|
206
|
+
|
|
207
|
+
// Decode column letters to match getColLetter()
|
|
208
|
+
// A=0..Z=25, AA=26, AB=27, etc.
|
|
209
|
+
let colIndex;
|
|
210
|
+
if (letters.length === 1) {
|
|
211
|
+
colIndex = letters.charCodeAt(0) - 65;
|
|
212
|
+
} else {
|
|
213
|
+
const first = letters.charCodeAt(0) - 65 + 1;
|
|
214
|
+
const second = letters.charCodeAt(1) - 65;
|
|
215
|
+
colIndex = (first * 26) + second;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const rowIndex = parseInt(rowStr, 10);
|
|
219
|
+
|
|
220
|
+
if (subColStr && subRowStr) {
|
|
221
|
+
// Fine grid logic: index into the global fine grid (25px spacing)
|
|
222
|
+
const subCol = parseInt(subColStr, 10);
|
|
223
|
+
const subRow = parseInt(subRowStr, 10);
|
|
224
|
+
const fineCol = (colIndex * 4) + subCol;
|
|
225
|
+
const fineRow = (rowIndex * 4) + subRow;
|
|
226
|
+
const fineX = FINE_START + fineCol * FINE_SPACING;
|
|
227
|
+
const fineY = FINE_START + fineRow * FINE_SPACING;
|
|
228
|
+
return { x: fineX, y: fineY, screenX: fineX, screenY: fineY };
|
|
229
|
+
} else {
|
|
230
|
+
// Coarse
|
|
231
|
+
const x = START_OFFSET + colIndex * COARSE_SPACING;
|
|
232
|
+
const y = START_OFFSET + rowIndex * COARSE_SPACING;
|
|
233
|
+
return { x, y, screenX: x, screenY: y };
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function drawLocalFineGrid() {
|
|
238
|
+
if (state.currentMode !== 'selection') return;
|
|
239
|
+
const { mouse, width, height } = state;
|
|
240
|
+
if (!mouse) return;
|
|
241
|
+
|
|
242
|
+
const baseCol = Math.round((mouse.x - FINE_START) / FINE_SPACING);
|
|
243
|
+
const baseRow = Math.round((mouse.y - FINE_START) / FINE_SPACING);
|
|
244
|
+
|
|
245
|
+
const minCol = baseCol - LOCAL_FINE_RADIUS;
|
|
246
|
+
const maxCol = baseCol + LOCAL_FINE_RADIUS;
|
|
247
|
+
const minRow = baseRow - LOCAL_FINE_RADIUS;
|
|
248
|
+
const maxRow = baseRow + LOCAL_FINE_RADIUS;
|
|
249
|
+
|
|
250
|
+
ctx.fillStyle = 'rgba(120, 200, 255, 0.7)';
|
|
251
|
+
ctx.strokeStyle = 'rgba(255, 255, 255, 0.75)';
|
|
252
|
+
ctx.lineWidth = 1;
|
|
253
|
+
|
|
254
|
+
ctx.beginPath();
|
|
255
|
+
for (let c = minCol; c <= maxCol; c++) {
|
|
256
|
+
const x = FINE_START + c * FINE_SPACING;
|
|
257
|
+
if (x < 0 || x > width) continue;
|
|
258
|
+
for (let r = minRow; r <= maxRow; r++) {
|
|
259
|
+
const y = FINE_START + r * FINE_SPACING;
|
|
260
|
+
if (y < 0 || y > height) continue;
|
|
261
|
+
ctx.moveTo(x + 2, y);
|
|
262
|
+
ctx.arc(x, y, 2, 0, Math.PI * 2);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
ctx.fill();
|
|
266
|
+
ctx.stroke();
|
|
267
|
+
|
|
268
|
+
const centerX = FINE_START + baseCol * FINE_SPACING;
|
|
269
|
+
const centerY = FINE_START + baseRow * FINE_SPACING;
|
|
270
|
+
if (centerX >= 0 && centerX <= width && centerY >= 0 && centerY <= height) {
|
|
271
|
+
ctx.beginPath();
|
|
272
|
+
ctx.arc(centerX, centerY, 4, 0, Math.PI * 2);
|
|
273
|
+
ctx.strokeStyle = 'rgba(0, 255, 200, 0.9)';
|
|
274
|
+
ctx.lineWidth = 2;
|
|
275
|
+
ctx.stroke();
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// ===== INPUT HANDLING =====
|
|
280
|
+
|
|
281
|
+
// Visual Feedback Helper
|
|
282
|
+
function showFeedback(text) {
|
|
283
|
+
const el = document.getElementById('key-feedback');
|
|
284
|
+
let fb = el;
|
|
285
|
+
if(!fb) {
|
|
286
|
+
fb = document.createElement('div');
|
|
287
|
+
fb.id = 'key-feedback';
|
|
288
|
+
fb.style.cssText = `position:fixed; top:50%; left:50%; transform:translate(-50%,-50%);
|
|
289
|
+
background:rgba(0,120,215,0.9); color:white; padding:16px 32px; border-radius:8px;
|
|
290
|
+
font-size:18px; font-weight:600; opacity:0; transition:opacity 0.2s; pointer-events:none; z-index:99999;`;
|
|
291
|
+
document.body.appendChild(fb);
|
|
292
|
+
}
|
|
293
|
+
fb.textContent = text;
|
|
294
|
+
fb.style.opacity = 1;
|
|
295
|
+
clearTimeout(state.indicators.feedback.timeout);
|
|
296
|
+
state.indicators.feedback.timeout = setTimeout(() => fb.style.opacity = 0, 1000);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// Mouse Tracking for Virtual Interaction
|
|
300
|
+
document.addEventListener('mousemove', (e) => {
|
|
301
|
+
state.mouse = { x: e.clientX, y: e.clientY };
|
|
302
|
+
if(ui.coordsStatus) ui.coordsStatus.textContent = `${e.clientX}, ${e.clientY}`;
|
|
303
|
+
|
|
304
|
+
if (state.currentMode === 'selection') {
|
|
305
|
+
requestDraw();
|
|
306
|
+
// Virtual Interaction Logic
|
|
307
|
+
// Find nearest grid point
|
|
308
|
+
const spacing = state.zoomLevel >= 2 ? FINE_SPACING : COARSE_SPACING;
|
|
309
|
+
const offset = state.zoomLevel >= 2 ? FINE_START : START_OFFSET;
|
|
310
|
+
|
|
311
|
+
// Nearest index
|
|
312
|
+
const c = Math.round((e.clientX - offset) / spacing);
|
|
313
|
+
const r = Math.round((e.clientY - offset) / spacing);
|
|
314
|
+
const snapX = offset + c * spacing;
|
|
315
|
+
const snapY = offset + r * spacing;
|
|
316
|
+
|
|
317
|
+
// Dist
|
|
318
|
+
const dx = e.clientX - snapX;
|
|
319
|
+
const dy = e.clientY - snapY;
|
|
320
|
+
const dist = Math.sqrt(dx*dx + dy*dy);
|
|
321
|
+
|
|
322
|
+
// Highlight if close
|
|
323
|
+
if (dist < 30) {
|
|
324
|
+
if(ui.interactionRegion) {
|
|
325
|
+
ui.interactionRegion.style.left = (snapX - 15) + 'px';
|
|
326
|
+
ui.interactionRegion.style.top = (snapY - 15) + 'px';
|
|
327
|
+
ui.interactionRegion.style.width = '30px';
|
|
328
|
+
ui.interactionRegion.style.height = '30px';
|
|
329
|
+
ui.interactionRegion.classList.add('visible');
|
|
330
|
+
ui.interactionRegion.dataset.x = snapX;
|
|
331
|
+
ui.interactionRegion.dataset.y = snapY;
|
|
332
|
+
}
|
|
333
|
+
} else {
|
|
334
|
+
if(ui.interactionRegion) ui.interactionRegion.classList.remove('visible');
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
document.addEventListener('click', (e) => {
|
|
340
|
+
if (state.currentMode === 'selection' && ui.interactionRegion && ui.interactionRegion.classList.contains('visible')) {
|
|
341
|
+
const x = parseFloat(ui.interactionRegion.dataset.x);
|
|
342
|
+
const y = parseFloat(ui.interactionRegion.dataset.y);
|
|
343
|
+
|
|
344
|
+
// Flash effect
|
|
345
|
+
showPulse(x, y);
|
|
346
|
+
|
|
347
|
+
// Send to main
|
|
348
|
+
let label;
|
|
349
|
+
let type;
|
|
350
|
+
if (state.zoomLevel >= 2) {
|
|
351
|
+
const fineCol = Math.round((x - FINE_START) / FINE_SPACING);
|
|
352
|
+
const fineRow = Math.round((y - FINE_START) / FINE_SPACING);
|
|
353
|
+
label = generateLabel(fineCol, fineRow, true);
|
|
354
|
+
type = 'fine';
|
|
355
|
+
} else {
|
|
356
|
+
const colInit = Math.round((x - START_OFFSET) / COARSE_SPACING);
|
|
357
|
+
const rowInit = Math.round((y - START_OFFSET) / COARSE_SPACING);
|
|
358
|
+
label = generateLabel(colInit, rowInit, false);
|
|
359
|
+
type = 'coarse';
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
if(window.electronAPI) {
|
|
363
|
+
window.electronAPI.selectDot({
|
|
364
|
+
id: `virtual-${x}-${y}`,
|
|
365
|
+
x, y, bg: true, label,
|
|
366
|
+
screenX: x, screenY: y,
|
|
367
|
+
type
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
// Pulse Effect (Doppler)
|
|
374
|
+
function showPulse(x, y) {
|
|
375
|
+
const el = document.createElement('div');
|
|
376
|
+
el.className = 'pulse-ring';
|
|
377
|
+
el.style.cssText = `position:fixed; left:${x}px; top:${y}px; width:10px; height:10px;
|
|
378
|
+
transform:translate(-50%,-50%); background:rgba(0,255,200,0.5); border-radius:50%;
|
|
379
|
+
box-shadow: 0 0 15px rgba(0,255,200,0.8); border: 2px solid #00ffcc;
|
|
380
|
+
transition:all 0.6s cubic-bezier(0.25, 0.46, 0.45, 0.94); pointer-events:none; z-index:2147483647;`;
|
|
381
|
+
document.body.appendChild(el);
|
|
382
|
+
requestAnimationFrame(() => {
|
|
383
|
+
el.style.width = '120px';
|
|
384
|
+
el.style.height = '120px';
|
|
385
|
+
el.style.opacity = 0;
|
|
386
|
+
el.style.borderWidth = '0px';
|
|
387
|
+
});
|
|
388
|
+
setTimeout(() => el.remove(), 700);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// ===== IPC & COMMANDS =====
|
|
392
|
+
if (window.electronAPI) {
|
|
393
|
+
window.electronAPI.onModeChanged((mode) => {
|
|
394
|
+
state.currentMode = mode;
|
|
395
|
+
state.zoomLevel = 1;
|
|
396
|
+
|
|
397
|
+
if (mode === 'selection') {
|
|
398
|
+
if(ui.modeIndicator) ui.modeIndicator.classList.add('visible');
|
|
399
|
+
if(ui.border) ui.border.classList.add('active');
|
|
400
|
+
} else {
|
|
401
|
+
if(ui.modeIndicator) ui.modeIndicator.classList.remove('visible');
|
|
402
|
+
if(ui.border) ui.border.classList.remove('active');
|
|
403
|
+
if(ui.interactionRegion) ui.interactionRegion.classList.remove('visible');
|
|
404
|
+
}
|
|
405
|
+
requestDraw();
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
window.electronAPI.onOverlayCommand((data) => {
|
|
409
|
+
handleCommand(data);
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
// Initialize State from Main Process
|
|
413
|
+
window.electronAPI.getState().then(initialState => {
|
|
414
|
+
console.log('Initial state loaded:', initialState);
|
|
415
|
+
if (initialState.overlayMode) {
|
|
416
|
+
state.currentMode = initialState.overlayMode;
|
|
417
|
+
// If valid mode, trigger UI update
|
|
418
|
+
if (state.currentMode === 'selection') {
|
|
419
|
+
if(ui.modeIndicator) ui.modeIndicator.classList.add('visible');
|
|
420
|
+
if(ui.border) ui.border.classList.add('active');
|
|
421
|
+
}
|
|
422
|
+
requestDraw();
|
|
423
|
+
}
|
|
424
|
+
// Load inspect mode state if available
|
|
425
|
+
if (initialState.inspectMode !== undefined) {
|
|
426
|
+
state.inspectMode = initialState.inspectMode;
|
|
427
|
+
updateInspectIndicator();
|
|
428
|
+
}
|
|
429
|
+
}).catch(err => console.error('Failed to get initial state:', err));
|
|
430
|
+
|
|
431
|
+
// Listen for inspect regions update
|
|
432
|
+
if (window.electronAPI.onInspectRegionsUpdate) {
|
|
433
|
+
window.electronAPI.onInspectRegionsUpdate((regions) => {
|
|
434
|
+
console.log('Received inspect regions:', regions?.length || 0);
|
|
435
|
+
updateInspectRegions(regions);
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// Listen for inspect mode toggle
|
|
440
|
+
if (window.electronAPI.onInspectModeChanged) {
|
|
441
|
+
window.electronAPI.onInspectModeChanged((enabled) => {
|
|
442
|
+
console.log('Inspect mode changed:', enabled);
|
|
443
|
+
state.inspectMode = enabled;
|
|
444
|
+
updateInspectIndicator();
|
|
445
|
+
if (!enabled) {
|
|
446
|
+
clearInspectRegions();
|
|
447
|
+
}
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
// Identify
|
|
452
|
+
console.log('Hooked electronAPI events');
|
|
453
|
+
} else {
|
|
454
|
+
console.warn('electronAPI not found - running in standalone mode?');
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function handleCommand(data) {
|
|
458
|
+
console.log('Command:', data.action);
|
|
459
|
+
switch (data.action) {
|
|
460
|
+
case 'toggle-fine':
|
|
461
|
+
state.zoomLevel = state.zoomLevel >= 2 ? 1 : 2;
|
|
462
|
+
showFeedback(state.zoomLevel >= 2 ? 'Fine Grid ON' : 'Fine Grid OFF');
|
|
463
|
+
requestDraw();
|
|
464
|
+
break;
|
|
465
|
+
case 'show-all':
|
|
466
|
+
state.zoomLevel = 3;
|
|
467
|
+
showFeedback('All Grids Visible');
|
|
468
|
+
requestDraw();
|
|
469
|
+
break;
|
|
470
|
+
case 'zoom-in':
|
|
471
|
+
state.zoomLevel = Math.min(3, state.zoomLevel + 1);
|
|
472
|
+
showFeedback(`Zoom: ${state.zoomLevel}x`);
|
|
473
|
+
requestDraw();
|
|
474
|
+
break;
|
|
475
|
+
case 'zoom-out':
|
|
476
|
+
state.zoomLevel = Math.max(1, state.zoomLevel - 1);
|
|
477
|
+
showFeedback(`Zoom: ${state.zoomLevel}x`);
|
|
478
|
+
requestDraw();
|
|
479
|
+
break;
|
|
480
|
+
case 'set-click-through':
|
|
481
|
+
document.body.style.pointerEvents = data.enabled ? 'none' : '';
|
|
482
|
+
if(ui.interactionRegion) ui.interactionRegion.style.pointerEvents = data.enabled ? 'none' : '';
|
|
483
|
+
// Also update inspect regions pointer events
|
|
484
|
+
if(ui.inspectContainer) ui.inspectContainer.style.pointerEvents = data.enabled ? 'none' : '';
|
|
485
|
+
break;
|
|
486
|
+
case 'pulse-click':
|
|
487
|
+
case 'highlight-coordinate':
|
|
488
|
+
showPulse(data.x, data.y);
|
|
489
|
+
break;
|
|
490
|
+
case 'get-coordinates':
|
|
491
|
+
if (data.label && window.electronAPI.sendCoordinates) {
|
|
492
|
+
// Not implemented in preload yet, but logical place
|
|
493
|
+
// For now, we rely on main process calculating it via ai-service
|
|
494
|
+
}
|
|
495
|
+
break;
|
|
496
|
+
// Inspect mode commands
|
|
497
|
+
case 'toggle-inspect':
|
|
498
|
+
state.inspectMode = !state.inspectMode;
|
|
499
|
+
showFeedback(state.inspectMode ? 'Inspect Mode ON' : 'Inspect Mode OFF');
|
|
500
|
+
updateInspectIndicator();
|
|
501
|
+
if (!state.inspectMode) {
|
|
502
|
+
clearInspectRegions();
|
|
503
|
+
}
|
|
504
|
+
break;
|
|
505
|
+
case 'update-inspect-regions':
|
|
506
|
+
if (data.regions) {
|
|
507
|
+
updateInspectRegions(data.regions);
|
|
508
|
+
}
|
|
509
|
+
break;
|
|
510
|
+
case 'clear-inspect-regions':
|
|
511
|
+
clearInspectRegions();
|
|
512
|
+
break;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
if (ui.gridStatus) {
|
|
516
|
+
ui.gridStatus.textContent = state.zoomLevel > 1 ? 'Grid: Fine' : 'Grid: Coarse';
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// ===== INSPECT MODE FUNCTIONS =====
|
|
521
|
+
|
|
522
|
+
/**
|
|
523
|
+
* Update inspect indicator visibility
|
|
524
|
+
*/
|
|
525
|
+
function updateInspectIndicator() {
|
|
526
|
+
if (ui.inspectIndicator) {
|
|
527
|
+
if (state.inspectMode) {
|
|
528
|
+
ui.inspectIndicator.classList.add('visible');
|
|
529
|
+
} else {
|
|
530
|
+
ui.inspectIndicator.classList.remove('visible');
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* Update inspect regions display
|
|
537
|
+
* @param {Array} regions - Array of region objects with bounds, label, role, confidence
|
|
538
|
+
*/
|
|
539
|
+
function updateInspectRegions(regions) {
|
|
540
|
+
if (!ui.inspectContainer) return;
|
|
541
|
+
|
|
542
|
+
// Clear existing regions
|
|
543
|
+
ui.inspectContainer.innerHTML = '';
|
|
544
|
+
state.inspectRegions = regions || [];
|
|
545
|
+
|
|
546
|
+
// Update region count
|
|
547
|
+
if (ui.regionCount) {
|
|
548
|
+
ui.regionCount.textContent = state.inspectRegions.length;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
// Render regions
|
|
552
|
+
state.inspectRegions.forEach((region, index) => {
|
|
553
|
+
const el = createRegionElement(region, index);
|
|
554
|
+
ui.inspectContainer.appendChild(el);
|
|
555
|
+
});
|
|
556
|
+
|
|
557
|
+
console.log(`Rendered ${state.inspectRegions.length} inspect regions`);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Create a DOM element for an inspect region
|
|
562
|
+
* @param {Object} region - Region data
|
|
563
|
+
* @param {number} index - Region index
|
|
564
|
+
* @returns {HTMLElement}
|
|
565
|
+
*/
|
|
566
|
+
function createRegionElement(region, index) {
|
|
567
|
+
const el = document.createElement('div');
|
|
568
|
+
el.className = 'inspect-region';
|
|
569
|
+
el.dataset.regionId = region.id;
|
|
570
|
+
el.dataset.index = index;
|
|
571
|
+
|
|
572
|
+
// Position and size
|
|
573
|
+
const bounds = region.bounds || {};
|
|
574
|
+
el.style.left = `${bounds.x || 0}px`;
|
|
575
|
+
el.style.top = `${bounds.y || 0}px`;
|
|
576
|
+
el.style.width = `${bounds.width || 0}px`;
|
|
577
|
+
el.style.height = `${bounds.height || 0}px`;
|
|
578
|
+
|
|
579
|
+
// Add classes for state
|
|
580
|
+
// Handle undefined/null confidence - default to 1.0 (high confidence)
|
|
581
|
+
const confidence = region.confidence ?? 1.0;
|
|
582
|
+
if (confidence < 0.7) {
|
|
583
|
+
el.classList.add('low-confidence');
|
|
584
|
+
}
|
|
585
|
+
if (region.id === state.selectedRegionId) {
|
|
586
|
+
el.classList.add('selected');
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
// Add label
|
|
590
|
+
const label = document.createElement('span');
|
|
591
|
+
label.className = 'inspect-region-label';
|
|
592
|
+
label.textContent = region.label || region.role || `Region ${index + 1}`;
|
|
593
|
+
el.appendChild(label);
|
|
594
|
+
|
|
595
|
+
// Event handlers
|
|
596
|
+
el.addEventListener('mouseenter', (e) => {
|
|
597
|
+
state.hoveredRegion = region;
|
|
598
|
+
showInspectTooltip(region, e.clientX, e.clientY);
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
el.addEventListener('mouseleave', () => {
|
|
602
|
+
state.hoveredRegion = null;
|
|
603
|
+
hideInspectTooltip();
|
|
604
|
+
});
|
|
605
|
+
|
|
606
|
+
el.addEventListener('mousemove', (e) => {
|
|
607
|
+
if (state.hoveredRegion === region) {
|
|
608
|
+
positionTooltip(e.clientX, e.clientY);
|
|
609
|
+
}
|
|
610
|
+
});
|
|
611
|
+
|
|
612
|
+
el.addEventListener('click', (e) => {
|
|
613
|
+
e.stopPropagation();
|
|
614
|
+
selectRegion(region);
|
|
615
|
+
});
|
|
616
|
+
|
|
617
|
+
return el;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* Show inspect tooltip for a region
|
|
622
|
+
* @param {Object} region - Region data
|
|
623
|
+
* @param {number} x - Mouse X position
|
|
624
|
+
* @param {number} y - Mouse Y position
|
|
625
|
+
*/
|
|
626
|
+
function showInspectTooltip(region, x, y) {
|
|
627
|
+
if (!ui.inspectTooltip) return;
|
|
628
|
+
|
|
629
|
+
// Update tooltip content
|
|
630
|
+
const roleEl = ui.inspectTooltip.querySelector('.tooltip-role');
|
|
631
|
+
const labelEl = ui.inspectTooltip.querySelector('.tooltip-label');
|
|
632
|
+
const textEl = document.getElementById('tooltip-text');
|
|
633
|
+
const posEl = document.getElementById('tooltip-position');
|
|
634
|
+
const confEl = document.getElementById('tooltip-confidence');
|
|
635
|
+
const confBar = document.getElementById('tooltip-confidence-bar');
|
|
636
|
+
|
|
637
|
+
if (roleEl) roleEl.textContent = region.role || 'element';
|
|
638
|
+
if (labelEl) labelEl.textContent = region.label || 'Unknown';
|
|
639
|
+
if (textEl) textEl.textContent = region.text || '-';
|
|
640
|
+
|
|
641
|
+
const centerX = Math.round((region.bounds?.x || 0) + (region.bounds?.width || 0) / 2);
|
|
642
|
+
const centerY = Math.round((region.bounds?.y || 0) + (region.bounds?.height || 0) / 2);
|
|
643
|
+
if (posEl) posEl.textContent = `${centerX}, ${centerY}`;
|
|
644
|
+
|
|
645
|
+
const confidence = Math.round((region.confidence || 0.5) * 100);
|
|
646
|
+
if (confEl) confEl.textContent = `${confidence}%`;
|
|
647
|
+
if (confBar) confBar.style.width = `${confidence}%`;
|
|
648
|
+
|
|
649
|
+
// Position and show tooltip
|
|
650
|
+
positionTooltip(x, y);
|
|
651
|
+
ui.inspectTooltip.classList.add('visible');
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/**
|
|
655
|
+
* Position tooltip near cursor
|
|
656
|
+
* @param {number} x - Mouse X
|
|
657
|
+
* @param {number} y - Mouse Y
|
|
658
|
+
*/
|
|
659
|
+
function positionTooltip(x, y) {
|
|
660
|
+
if (!ui.inspectTooltip) return;
|
|
661
|
+
|
|
662
|
+
const offset = 15;
|
|
663
|
+
const tooltipRect = ui.inspectTooltip.getBoundingClientRect();
|
|
664
|
+
|
|
665
|
+
// Default position: below and to the right of cursor
|
|
666
|
+
let left = x + offset;
|
|
667
|
+
let top = y + offset;
|
|
668
|
+
|
|
669
|
+
// Adjust if tooltip would go off screen
|
|
670
|
+
if (left + tooltipRect.width > window.innerWidth) {
|
|
671
|
+
left = x - tooltipRect.width - offset;
|
|
672
|
+
}
|
|
673
|
+
if (top + tooltipRect.height > window.innerHeight) {
|
|
674
|
+
top = y - tooltipRect.height - offset;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
ui.inspectTooltip.style.left = `${left}px`;
|
|
678
|
+
ui.inspectTooltip.style.top = `${top}px`;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Hide inspect tooltip
|
|
683
|
+
*/
|
|
684
|
+
function hideInspectTooltip() {
|
|
685
|
+
if (ui.inspectTooltip) {
|
|
686
|
+
ui.inspectTooltip.classList.remove('visible');
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* Select a region and notify main process
|
|
692
|
+
* @param {Object} region - Region to select
|
|
693
|
+
*/
|
|
694
|
+
function selectRegion(region) {
|
|
695
|
+
// Update state
|
|
696
|
+
state.selectedRegionId = region.id;
|
|
697
|
+
|
|
698
|
+
// Update visual state
|
|
699
|
+
document.querySelectorAll('.inspect-region').forEach(el => {
|
|
700
|
+
el.classList.remove('selected');
|
|
701
|
+
if (el.dataset.regionId === region.id) {
|
|
702
|
+
el.classList.add('selected');
|
|
703
|
+
}
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
// Show pulse at region center
|
|
707
|
+
const centerX = (region.bounds?.x || 0) + (region.bounds?.width || 0) / 2;
|
|
708
|
+
const centerY = (region.bounds?.y || 0) + (region.bounds?.height || 0) / 2;
|
|
709
|
+
showPulse(centerX, centerY);
|
|
710
|
+
|
|
711
|
+
// Notify main process
|
|
712
|
+
if (window.electronAPI?.selectInspectRegion) {
|
|
713
|
+
window.electronAPI.selectInspectRegion({
|
|
714
|
+
targetId: region.id,
|
|
715
|
+
region: region,
|
|
716
|
+
bounds: region.bounds,
|
|
717
|
+
x: centerX,
|
|
718
|
+
y: centerY
|
|
719
|
+
});
|
|
720
|
+
} else if (window.electronAPI?.selectDot) {
|
|
721
|
+
// Fallback to dot selection
|
|
722
|
+
window.electronAPI.selectDot({
|
|
723
|
+
id: `inspect-${region.id}`,
|
|
724
|
+
x: centerX,
|
|
725
|
+
y: centerY,
|
|
726
|
+
label: region.label || region.role,
|
|
727
|
+
targetId: region.id,
|
|
728
|
+
type: 'inspect-region',
|
|
729
|
+
screenX: centerX,
|
|
730
|
+
screenY: centerY,
|
|
731
|
+
region: region
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
showFeedback(`Selected: ${region.label || region.role || 'Region'}`);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* Clear all inspect regions
|
|
740
|
+
*/
|
|
741
|
+
function clearInspectRegions() {
|
|
742
|
+
if (ui.inspectContainer) {
|
|
743
|
+
ui.inspectContainer.innerHTML = '';
|
|
744
|
+
}
|
|
745
|
+
state.inspectRegions = [];
|
|
746
|
+
state.hoveredRegion = null;
|
|
747
|
+
state.selectedRegionId = null;
|
|
748
|
+
|
|
749
|
+
if (ui.regionCount) {
|
|
750
|
+
ui.regionCount.textContent = '0';
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
hideInspectTooltip();
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* Find region at a point (for hover detection)
|
|
758
|
+
* Uses exclusive bounds (x < right, y < bottom) for correct hit detection
|
|
759
|
+
* @param {number} x - X coordinate
|
|
760
|
+
* @param {number} y - Y coordinate
|
|
761
|
+
* @returns {Object|null}
|
|
762
|
+
*/
|
|
763
|
+
function findRegionAtPoint(x, y) {
|
|
764
|
+
for (const region of state.inspectRegions) {
|
|
765
|
+
const b = region.bounds;
|
|
766
|
+
// Use exclusive bounds (< instead of <=) for mathematical correctness
|
|
767
|
+
if (x >= b.x && x < b.x + b.width && y >= b.y && y < b.y + b.height) {
|
|
768
|
+
return region;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
return null;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
// Expose Helper Global
|
|
775
|
+
window.labelToScreenCoordinates = labelToScreenCoordinates;
|
|
776
|
+
|
|
777
|
+
// Expose inspect functions globally for debugging
|
|
778
|
+
window.updateInspectRegions = updateInspectRegions;
|
|
779
|
+
window.clearInspectRegions = clearInspectRegions;
|
|
780
|
+
|
|
781
|
+
console.log('High-Performance Canvas Overlay Loaded');
|
|
782
|
+
requestDraw();
|