copilot-liku-cli 0.0.1

package/src/renderer/overlay/preload.js

@@ -0,0 +1,90 @@
+const { contextBridge, ipcRenderer } = require('electron');
+
+// BLOCKER-1 FIX: Inline grid-math since sandboxed preload can't access 'path' module
+// or require external files. These are small utilities needed for coordinate conversion.
+
+const COARSE_SPACING = 100;
+const FINE_SPACING = 25;
+const START_OFFSET = COARSE_SPACING / 2;
+const FINE_START = FINE_SPACING / 2;
+
+function colLettersToIndex(letters) {
+  if (!letters || letters.length === 0) return null;
+  if (letters.length === 1) {
+    return letters.charCodeAt(0) - 65;
+  }
+  const first = letters.charCodeAt(0) - 65 + 1;
+  const second = letters.charCodeAt(1) - 65;
+  return (first * 26) + second;
+}
+
+function labelToScreenCoordinates(label) {
+  if (!label) return null;
+  const match = label.match(/^([A-Z]+)(\d+)(\.(\d)(\d))?$/);
+  if (!match) return null;
+
+  const [, letters, rowStr, , subColStr, subRowStr] = match;
+  const colIndex = colLettersToIndex(letters);
+  const rowIndex = parseInt(rowStr, 10);
+  if (colIndex === null || Number.isNaN(rowIndex)) return null;
+
+  if (subColStr && subRowStr) {
+    const subCol = parseInt(subColStr, 10);
+    const subRow = parseInt(subRowStr, 10);
+    if (Number.isNaN(subCol) || Number.isNaN(subRow)) return null;
+    const fineCol = (colIndex * 4) + subCol;
+    const fineRow = (rowIndex * 4) + subRow;
+    const x = FINE_START + fineCol * FINE_SPACING;
+    const y = FINE_START + fineRow * FINE_SPACING;
+    return { x, y, screenX: x, screenY: y, colIndex, rowIndex, fineCol, fineRow, subCol, subRow, isFine: true };
+  }
+
+  const x = START_OFFSET + colIndex * COARSE_SPACING;
+  const y = START_OFFSET + rowIndex * COARSE_SPACING;
+  return { x, y, screenX: x, screenY: y, colIndex, rowIndex, isFine: false };
+}
+
+const gridConstants = {
+  coarseSpacing: COARSE_SPACING,
+  fineSpacing: FINE_SPACING,
+  startOffset: START_OFFSET,
+  fineStart: FINE_START,
+  localFineRadius: 3
+};
+
+// Expose protected methods that allow the renderer process to use
+// the ipcRenderer without exposing the entire object
+contextBridge.exposeInMainWorld('electronAPI', {
+  // Send dot selection to main process
+  selectDot: (data) => ipcRenderer.send('dot-selected', data),
+  
+  // Listen for mode changes
+  onModeChanged: (callback) => ipcRenderer.on('mode-changed', (event, mode) => callback(mode)),
+  
+  // Listen for overlay commands (keyboard shortcuts routed via main process)
+  onOverlayCommand: (callback) => ipcRenderer.on('overlay-command', (event, data) => callback(data)),
+  
+  // Get current state
+  getState: () => ipcRenderer.invoke('get-state'),
+
+  // Grid math helpers (inlined above)
+  getGridConstants: () => gridConstants,
+  labelToScreenCoordinates: (label) => labelToScreenCoordinates(label),
+  
+  // ===== INSPECT MODE API =====
+  
+  // Select an inspect region (sends targetId + bounds to main)
+  selectInspectRegion: (data) => ipcRenderer.send('inspect-region-selected', data),
+  
+  // Listen for inspect regions updates
+  onInspectRegionsUpdate: (callback) => ipcRenderer.on('inspect-regions-update', (event, regions) => callback(regions)),
+  
+  // Listen for inspect mode toggle
+  onInspectModeChanged: (callback) => ipcRenderer.on('inspect-mode-changed', (event, enabled) => callback(enabled)),
+  
+  // Request inspect region detection
+  requestInspectRegions: () => ipcRenderer.send('request-inspect-regions'),
+  
+  // Toggle inspect mode
+  toggleInspectMode: () => ipcRenderer.send('toggle-inspect-mode')
+});

package/src/shared/grid-math.js

@@ -0,0 +1,82 @@
+const COARSE_SPACING = 100;
+const FINE_SPACING = 25;
+const START_OFFSET = COARSE_SPACING / 2;
+const FINE_START = FINE_SPACING / 2;
+
+function getColLetter(colIndex) {
+  let letter = '';
+  if (colIndex >= 26) {
+    letter += String.fromCharCode(65 + Math.floor(colIndex / 26) - 1);
+  }
+  letter += String.fromCharCode(65 + (colIndex % 26));
+  return letter;
+}
+
+function colLettersToIndex(letters) {
+  if (!letters || letters.length === 0) return null;
+  if (letters.length === 1) {
+    return letters.charCodeAt(0) - 65;
+  }
+  const first = letters.charCodeAt(0) - 65 + 1;
+  const second = letters.charCodeAt(1) - 65;
+  return (first * 26) + second;
+}
+
+function labelToScreenCoordinates(label) {
+  if (!label) return null;
+  const match = label.match(/^([A-Z]+)(\d+)(\.(\d)(\d))?$/);
+  if (!match) return null;
+
+  const [, letters, rowStr, , subColStr, subRowStr] = match;
+  const colIndex = colLettersToIndex(letters);
+  const rowIndex = parseInt(rowStr, 10);
+  if (colIndex === null || Number.isNaN(rowIndex)) return null;
+
+  if (subColStr && subRowStr) {
+    const subCol = parseInt(subColStr, 10);
+    const subRow = parseInt(subRowStr, 10);
+    if (Number.isNaN(subCol) || Number.isNaN(subRow)) return null;
+    const fineCol = (colIndex * 4) + subCol;
+    const fineRow = (rowIndex * 4) + subRow;
+    const x = FINE_START + fineCol * FINE_SPACING;
+    const y = FINE_START + fineRow * FINE_SPACING;
+    return {
+      x,
+      y,
+      screenX: x,
+      screenY: y,
+      colIndex,
+      rowIndex,
+      fineCol,
+      fineRow,
+      subCol,
+      subRow,
+      isFine: true
+    };
+  }
+
+  const x = START_OFFSET + colIndex * COARSE_SPACING;
+  const y = START_OFFSET + rowIndex * COARSE_SPACING;
+  return {
+    x,
+    y,
+    screenX: x,
+    screenY: y,
+    colIndex,
+    rowIndex,
+    isFine: false
+  };
+}
+
+module.exports = {
+  constants: {
+    coarseSpacing: COARSE_SPACING,
+    fineSpacing: FINE_SPACING,
+    startOffset: START_OFFSET,
+    fineStart: FINE_START,
+    localFineRadius: 3
+  },
+  getColLetter,
+  colLettersToIndex,
+  labelToScreenCoordinates
+};

package/src/shared/inspect-types.js

@@ -0,0 +1,230 @@
+/**
+ * Inspect Overlay Data Contracts
+ * Shared type definitions for inspect regions, window context, and action traces
+ */
+
+/**
+ * Inspect Region Data Contract
+ * Represents an actionable region on screen detected through various sources
+ * @typedef {Object} InspectRegion
+ * @property {string} id - Unique identifier for the region
+ * @property {Object} bounds - Bounding box {x, y, width, height}
+ * @property {string} label - Human-readable label (e.g., "Search button")
+ * @property {string} text - Text content if available
+ * @property {string} role - Accessibility role (button, textbox, etc.)
+ * @property {number} confidence - Detection confidence 0-1
+ * @property {string} source - Detection source (accessibility, ocr, heuristic)
+ * @property {number} timestamp - When this region was detected
+ */
+
+/**
+ * Window Context Data Contract
+ * Information about the active window and process
+ * @typedef {Object} WindowContext
+ * @property {string} appName - Application name
+ * @property {string} windowTitle - Window title
+ * @property {number} pid - Process ID
+ * @property {Object} bounds - Window bounds {x, y, width, height}
+ * @property {number} zOrder - Z-order (depth) of window
+ * @property {number} scaleFactor - Display scale factor for DPI normalization
+ */
+
+/**
+ * Action Trace Data Contract
+ * Records of actions for replay and debugging
+ * @typedef {Object} ActionTrace
+ * @property {string} actionId - Unique action identifier
+ * @property {string} type - Action type (click, type, key, etc.)
+ * @property {string} [targetId] - ID of target region if applicable
+ * @property {number} x - X coordinate
+ * @property {number} y - Y coordinate
+ * @property {number} timestamp - When action was executed
+ * @property {string} outcome - Result (success, failed, pending)
+ */
+
+/**
+ * Create a new inspect region object
+ * @param {Object} params - Region parameters
+ * @returns {InspectRegion}
+ */
+function createInspectRegion(params) {
+  return {
+    id: params.id || `region-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`,
+    bounds: {
+      x: params.x || params.bounds?.x || 0,
+      y: params.y || params.bounds?.y || 0,
+      width: params.width || params.bounds?.width || 0,
+      height: params.height || params.bounds?.height || 0
+    },
+    label: params.label || params.name || '',
+    text: params.text || '',
+    role: params.role || params.controlType || 'unknown',
+    confidence: typeof params.confidence === 'number' ? params.confidence : 0.5,
+    source: params.source || 'unknown',
+    timestamp: params.timestamp || Date.now()
+  };
+}
+
+/**
+ * Create a new window context object
+ * @param {Object} params - Window parameters
+ * @returns {WindowContext}
+ */
+function createWindowContext(params) {
+  return {
+    appName: params.appName || params.processName || '',
+    windowTitle: params.windowTitle || params.title || '',
+    pid: params.pid || params.processId || 0,
+    bounds: {
+      x: params.bounds?.x || params.bounds?.X || 0,
+      y: params.bounds?.y || params.bounds?.Y || 0,
+      width: params.bounds?.width || params.bounds?.Width || 0,
+      height: params.bounds?.height || params.bounds?.Height || 0
+    },
+    zOrder: params.zOrder || 0,
+    scaleFactor: params.scaleFactor || 1
+  };
+}
+
+/**
+ * Create a new action trace object
+ * @param {Object} params - Action parameters
+ * @returns {ActionTrace}
+ */
+function createActionTrace(params) {
+  return {
+    actionId: params.actionId || `action-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`,
+    type: params.type || 'unknown',
+    targetId: params.targetId || null,
+    x: params.x || 0,
+    y: params.y || 0,
+    timestamp: params.timestamp || Date.now(),
+    outcome: params.outcome || 'pending'
+  };
+}
+
+/**
+ * Normalize coordinates with scale factor
+ * @param {number} x - X coordinate
+ * @param {number} y - Y coordinate
+ * @param {number} scaleFactor - Display scale factor
+ * @returns {Object} Normalized {x, y}
+ */
+function normalizeCoordinates(x, y, scaleFactor = 1) {
+  return {
+    x: Math.round(x * scaleFactor),
+    y: Math.round(y * scaleFactor)
+  };
+}
+
+/**
+ * Denormalize coordinates from scaled to logical
+ * @param {number} x - X coordinate (scaled)
+ * @param {number} y - Y coordinate (scaled)
+ * @param {number} scaleFactor - Display scale factor
+ * @returns {Object} Logical {x, y}
+ */
+function denormalizeCoordinates(x, y, scaleFactor = 1) {
+  return {
+    x: Math.round(x / scaleFactor),
+    y: Math.round(y / scaleFactor)
+  };
+}
+
+/**
+ * Check if a point is within a region's bounds
+ * @param {number} x - X coordinate
+ * @param {number} y - Y coordinate
+ * @param {InspectRegion} region - The region to check
+ * @returns {boolean}
+ */
+function isPointInRegion(x, y, region) {
+  const { bounds } = region;
+  return x >= bounds.x && 
+         x < bounds.x + bounds.width && 
+         y >= bounds.y && 
+         y < bounds.y + bounds.height;
+}
+
+/**
+ * Find the closest region to a point
+ * @param {number} x - X coordinate
+ * @param {number} y - Y coordinate
+ * @param {InspectRegion[]} regions - Array of regions
+ * @returns {InspectRegion|null} Closest region or null
+ */
+function findClosestRegion(x, y, regions) {
+  if (!regions || regions.length === 0) return null;
+  
+  let closest = null;
+  let minDist = Infinity;
+  
+  for (const region of regions) {
+    const centerX = region.bounds.x + region.bounds.width / 2;
+    const centerY = region.bounds.y + region.bounds.height / 2;
+    const dist = Math.sqrt(Math.pow(x - centerX, 2) + Math.pow(y - centerY, 2));
+    
+    if (dist < minDist) {
+      minDist = dist;
+      closest = region;
+    }
+  }
+  
+  return closest;
+}
+
+/**
+ * Find region containing a point
+ * @param {number} x - X coordinate
+ * @param {number} y - Y coordinate
+ * @param {InspectRegion[]} regions - Array of regions
+ * @returns {InspectRegion|null} Containing region or null
+ */
+function findRegionAtPoint(x, y, regions) {
+  if (!regions || regions.length === 0) return null;
+  
+  // Find all regions containing the point
+  const containing = regions.filter(r => isPointInRegion(x, y, r));
+  
+  if (containing.length === 0) return null;
+  if (containing.length === 1) return containing[0];
+  
+  // If multiple regions, return the smallest (most specific)
+  return containing.reduce((smallest, r) => {
+    const smallestArea = smallest.bounds.width * smallest.bounds.height;
+    const rArea = r.bounds.width * r.bounds.height;
+    return rArea < smallestArea ? r : smallest;
+  });
+}
+
+/**
+ * Format region for AI context
+ * @param {InspectRegion} region - The region to format
+ * @returns {Object} AI-friendly format
+ */
+function formatRegionForAI(region) {
+  return {
+    id: region.id,
+    label: region.label,
+    text: region.text,
+    role: region.role,
+    confidence: region.confidence,
+    center: {
+      x: Math.round(region.bounds.x + region.bounds.width / 2),
+      y: Math.round(region.bounds.y + region.bounds.height / 2)
+    },
+    bounds: region.bounds
+  };
+}
+
+module.exports = {
+  createInspectRegion,
+  createWindowContext,
+  createActionTrace,
+  normalizeCoordinates,
+  denormalizeCoordinates,
+  isPointInRegion,
+  findClosestRegion,
+  findRegionAtPoint,
+  formatRegionForAI
+};