copilot-liku-cli 0.0.1

package/src/main/ui-automation/index.js

@@ -0,0 +1,164 @@
+/**
+ * UI Automation Module
+ * 
+ * Comprehensive Windows UI automation using semantic element discovery,
+ * SendInput API for reliable input, and PowerShell for system integration.
+ * 
+ * @module ui-automation
+ * 
+ * @example
+ * const ui = require('./ui-automation');
+ * 
+ * // Find and click a button by text
+ * await ui.click({ text: 'Submit' });
+ * 
+ * // Type in a text field
+ * await ui.click({ automationId: 'searchBox' });
+ * await ui.typeText('Hello world');
+ * 
+ * // Wait for element and click
+ * await ui.waitAndClick({ text: 'OK' }, { timeout: 5000 });
+ * 
+ * // Take screenshot
+ * await ui.screenshot({ path: 'capture.png' });
+ */
+
+// Configuration
+const { CONFIG, CONTROL_TYPES } = require('./config');
+
+// Core utilities
+const { sleep, debug, log, executePowerShellScript } = require('./core');
+
+// Element operations
+const { 
+  findElements, 
+  findElement, 
+  waitForElement, 
+  waitForElementGone 
+} = require('./elements');
+
+// Mouse operations
+const {
+  moveMouse,
+  getMousePosition,
+  clickAt,
+  doubleClickAt,
+  drag,
+  scroll,
+  scrollUp,
+  scrollDown,
+  scrollLeft,
+  scrollRight,
+} = require('./mouse');
+
+// Keyboard operations
+const {
+  typeText,
+  sendKeys,
+  keyDown,
+  keyUp,
+  VK,
+} = require('./keyboard');
+
+// Window operations
+const {
+  getActiveWindow,
+  findWindows,
+  focusWindow,
+  minimizeWindow,
+  maximizeWindow,
+  restoreWindow,
+} = require('./window');
+
+// High-level interactions
+const {
+  click,
+  clickByText,
+  clickByAutomationId,
+  rightClick,
+  doubleClick,
+  clickElement,
+  invokeElement,
+  fillField,
+  selectDropdownItem,
+  waitForWindow,
+  clickSequence,
+  hover,
+  waitAndClick,
+  clickAndWaitFor,
+  selectFromDropdown,
+} = require('./interactions');
+
+// Screenshot
+const {
+  screenshot,
+  screenshotActiveWindow,
+  screenshotElement,
+} = require('./screenshot');
+
+module.exports = {
+  // Configuration
+  CONFIG,
+  CONTROL_TYPES,
+  
+  // Core utilities
+  sleep,
+  debug,
+  log,
+  executePowerShellScript,
+  
+  // Element operations
+  findElements,
+  findElement,
+  waitForElement,
+  waitForElementGone,
+  
+  // Mouse operations - low level
+  moveMouse,
+  getMousePosition,
+  clickAt,
+  doubleClickAt,
+  drag,
+  scroll,
+  scrollUp,
+  scrollDown,
+  scrollLeft,
+  scrollRight,
+  
+  // Keyboard operations
+  typeText,
+  sendKeys,
+  keyDown,
+  keyUp,
+  VK,
+  
+  // Window operations
+  getActiveWindow,
+  findWindows,
+  focusWindow,
+  minimizeWindow,
+  maximizeWindow,
+  restoreWindow,
+  
+  // High-level interactions (element-based clicks)
+  click,
+  clickByText,
+  clickByAutomationId,
+  rightClick,
+  doubleClick,
+  clickElement,
+  invokeElement,
+  fillField,
+  selectDropdownItem,
+  waitForWindow,
+  clickSequence,
+  hover,
+  waitAndClick,
+  clickAndWaitFor,
+  selectFromDropdown,
+  
+  // Screenshot
+  screenshot,
+  screenshotActiveWindow,
+  screenshotElement,
+};

package/src/main/ui-automation/interactions/element-click.js

@@ -0,0 +1,211 @@
+/**
+ * Element Click Interactions
+ * 
+ * Click on UI elements by criteria (text, automationId, etc.)
+ * @module ui-automation/interactions/element-click
+ */
+
+const { findElement, waitForElement } = require('../elements');
+const { clickAt, doubleClickAt } = require('../mouse');
+const { executePowerShellScript } = require('../core/powershell');
+const { log, sleep } = require('../core/helpers');
+
+/**
+ * Click on an element found by criteria
+ * 
+ * @param {Object} criteria - Element search criteria
+ * @param {string} [criteria.text] - Element text/name
+ * @param {string} [criteria.automationId] - Automation ID
+ * @param {string} [criteria.controlType] - Control type
+ * @param {string} [criteria.className] - Class name
+ * @param {string} [criteria.windowTitle] - Window title to search in
+ * @param {Object} [options] - Click options
+ * @param {boolean} [options.doubleClick=false] - Double click instead
+ * @param {string} [options.button='left'] - Mouse button
+ * @param {boolean} [options.focusWindow=true] - Focus window first
+ * @param {number} [options.waitTimeout=0] - Wait for element (ms, 0 = no wait)
+ * @returns {Promise<{success: boolean, element: Object|null}>}
+ */
+async function click(criteria, options = {}) {
+  const { 
+    doubleClick = false, 
+    button = 'left',
+    focusWindow = true,
+    waitTimeout = 0,
+  } = options;
+  
+  // Find or wait for element
+  let findResult;
+  if (waitTimeout > 0) {
+    findResult = await waitForElement(criteria, { timeout: waitTimeout });
+  } else {
+    findResult = await findElement(criteria);
+  }
+  
+  // Extract element from result
+  const element = findResult?.element;
+  
+  if (!element || !element.bounds) {
+    log(`click: Element not found for criteria: ${JSON.stringify(criteria)}`, 'warn');
+    return { success: false, element: null, error: findResult?.error || 'Element not found' };
+  }
+  
+  // Calculate center point
+  const bounds = element.bounds;
+  const x = bounds.x + bounds.width / 2;
+  const y = bounds.y + bounds.height / 2;
+  
+  // Focus window if needed
+  if (focusWindow && element.windowHwnd) {
+    const { focusWindow: doFocus } = require('../window');
+    await doFocus(element.windowHwnd);
+    await sleep(50);
+  }
+  
+  // Click
+  const clickFn = doubleClick ? doubleClickAt : clickAt;
+  const clickResult = await clickFn(x, y, { button, focusWindow: false });
+  
+  log(`click on "${element.name || element.automationId}" at (${Math.round(x)}, ${Math.round(y)}) - ${clickResult.success ? 'success' : 'failed'}`);
+  
+  return { success: clickResult.success, element };
+}
+
+/**
+ * Click element by text
+ * 
+ * @param {string} text - Element text to find
+ * @param {Object} [options] - Click options
+ * @returns {Promise<{success: boolean, element: Object|null}>}
+ */
+async function clickByText(text, options = {}) {
+  return click({ text }, options);
+}
+
+/**
+ * Click element by automation ID
+ * 
+ * @param {string} automationId - Automation ID
+ * @param {Object} [options] - Click options
+ * @returns {Promise<{success: boolean, element: Object|null}>}
+ */
+async function clickByAutomationId(automationId, options = {}) {
+  return click({ automationId }, options);
+}
+
+/**
+ * Right-click on an element
+ * 
+ * @param {Object} criteria - Element search criteria
+ * @param {Object} [options] - Additional options
+ * @returns {Promise<{success: boolean, element: Object|null}>}
+ */
+async function rightClick(criteria, options = {}) {
+  return click(criteria, { ...options, button: 'right' });
+}
+
+/**
+ * Double-click on an element
+ * 
+ * @param {Object} criteria - Element search criteria
+ * @param {Object} [options] - Additional options
+ * @returns {Promise<{success: boolean, element: Object|null}>}
+ */
+async function doubleClick(criteria, options = {}) {
+  return click(criteria, { ...options, doubleClick: true });
+}
+
+/**
+ * Click on an element object directly (low-level)
+ * 
+ * @param {Object} element - Element with bounds property
+ * @param {Object} [options] - Click options
+ * @param {string} [options.button='left'] - Mouse button
+ * @param {boolean} [options.useInvoke=true] - Try Invoke pattern first
+ * @returns {Promise<{success: boolean, method: string, error?: string}>}
+ */
+async function clickElement(element, options = {}) {
+  const { button = 'left', useInvoke = true } = options;
+  
+  if (!element || !element.bounds) {
+    return { success: false, error: 'Invalid element' };
+  }
+  
+  const bounds = element.bounds;
+  const centerX = bounds.x + bounds.width / 2;
+  const centerY = bounds.y + bounds.height / 2;
+  
+  // Strategy 1: Try Invoke pattern for buttons
+  if (useInvoke && element.patterns?.includes('InvokePatternIdentifiers.Pattern')) {
+    log(`Attempting Invoke pattern for "${element.name}"`);
+    const invokeResult = await invokeElement(element);
+    if (invokeResult.success) {
+      return { success: true, method: 'invoke', element };
+    }
+  }
+  
+  // Strategy 2: Click
+  log(`Clicking "${element.name}" at (${centerX}, ${centerY})`);
+  const clickResult = await clickAt(centerX, centerY, { button, focusWindow: true });
+  
+  return {
+    success: clickResult.success,
+    method: 'sendInput',
+    element,
+    coordinates: clickResult.coordinates,
+  };
+}
+
+/**
+ * Invoke an element using UI Automation Invoke pattern
+ * Works directly with buttons without simulating mouse clicks
+ * 
+ * @param {Object} element - Element to invoke
+ * @returns {Promise<{success: boolean, error?: string}>}
+ */
+async function invokeElement(element) {
+  const searchName = (element.name || '').replace(/"/g, '`"');
+  
+  const psScript = `
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+
+$root = [System.Windows.Automation.AutomationElement]::RootElement
+$condition = [System.Windows.Automation.PropertyCondition]::new(
+    [System.Windows.Automation.AutomationElement]::NameProperty, 
+    "${searchName}"
+)
+$element = $root.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $condition)
+
+if ($element -eq $null) {
+    Write-Output '{"success": false, "error": "Element not found"}'
+    exit
+}
+
+try {
+    $invokePattern = $element.GetCurrentPattern([System.Windows.Automation.InvokePattern]::Pattern)
+    $invokePattern.Invoke()
+    Write-Output '{"success": true, "method": "invoke"}'
+} catch {
+    Write-Output "{\\"success\\": false, \\"error\\": \\"$($_.Exception.Message)\\"}"
+}
+`;
+
+  const result = await executePowerShellScript(psScript);
+  
+  try {
+    return JSON.parse(result.stdout.trim());
+  } catch {
+    return { success: false, error: result.error || 'Parse error' };
+  }
+}
+
+module.exports = {
+  click,
+  clickByText,
+  clickByAutomationId,
+  rightClick,
+  doubleClick,
+  clickElement,
+  invokeElement,
+};

package/src/main/ui-automation/interactions/high-level.js

@@ -0,0 +1,230 @@
+/**
+ * High-Level UI Interactions
+ * 
+ * Complex automation workflows and convenience functions.
+ * @module ui-automation/interactions/high-level
+ */
+
+const { findElement, findElements, waitForElement } = require('../elements');
+const { click, clickByText } = require('./element-click');
+const { typeText, sendKeys } = require('../keyboard');
+const { focusWindow, findWindows } = require('../window');
+const { log, sleep } = require('../core/helpers');
+
+/**
+ * Fill a text field by clicking it then typing
+ * 
+ * @param {Object} criteria - Element search criteria
+ * @param {string} text - Text to type
+ * @param {Object} [options] - Options
+ * @param {boolean} [options.clear=true] - Clear field first (Ctrl+A)
+ * @returns {Promise<{success: boolean}>}
+ */
+async function fillField(criteria, text, options = {}) {
+  const { clear = true } = options;
+  
+  // Click the field
+  const clickResult = await click(criteria);
+  if (!clickResult.success) {
+    return { success: false };
+  }
+  
+  await sleep(50);
+  
+  // Clear if requested
+  if (clear) {
+    await sendKeys('^a');
+    await sleep(20);
+  }
+  
+  // Type text
+  const typeResult = await typeText(text);
+  return { success: typeResult.success };
+}
+
+/**
+ * Select an item from a dropdown/combobox
+ * 
+ * @param {Object} dropdownCriteria - Criteria to find the dropdown
+ * @param {string|Object} itemCriteria - Item text or criteria
+ * @param {Object} [options] - Options
+ * @param {number} [options.itemWait=1000] - Time to wait for dropdown items to appear
+ * @returns {Promise<{success: boolean}>}
+ */
+async function selectDropdownItem(dropdownCriteria, itemCriteria, options = {}) {
+  const { itemWait = 1000 } = options;
+  
+  // Click dropdown to open
+  const openResult = await click(dropdownCriteria);
+  if (!openResult.success) {
+    log('selectDropdownItem: Failed to open dropdown', 'warn');
+    return { success: false };
+  }
+  
+  await sleep(itemWait);
+  
+  // Click item
+  const itemQuery = typeof itemCriteria === 'string' 
+    ? { text: itemCriteria } 
+    : itemCriteria;
+    
+  const itemResult = await click(itemQuery);
+  return { success: itemResult.success };
+}
+
+/**
+ * Wait for a window and focus it
+ * 
+ * @param {string|Object} criteria - Window title or search criteria
+ * @param {Object} [options] - Options
+ * @param {number} [options.timeout=10000] - Timeout in ms
+ * @param {number} [options.pollInterval=500] - Poll interval in ms
+ * @returns {Promise<{success: boolean, window: Object|null}>}
+ */
+async function waitForWindow(criteria, options = {}) {
+  const { timeout = 10000, pollInterval = 500 } = options;
+  const searchCriteria = typeof criteria === 'string' ? { title: criteria } : criteria;
+  
+  const startTime = Date.now();
+  
+  while (Date.now() - startTime < timeout) {
+    const windows = await findWindows(searchCriteria);
+    if (windows.length > 0) {
+      const result = await focusWindow(windows[0].hwnd);
+      return { success: result.success, window: windows[0] };
+    }
+    await sleep(pollInterval);
+  }
+  
+  log(`waitForWindow: Timeout waiting for window`, 'warn');
+  return { success: false, window: null };
+}
+
+/**
+ * Click a sequence of elements in order
+ * 
+ * @param {Array<Object>} steps - Array of {criteria, options?, delay?}
+ * @returns {Promise<{success: boolean, completedSteps: number}>}
+ */
+async function clickSequence(steps) {
+  let completedSteps = 0;
+  
+  for (const step of steps) {
+    const { criteria, options = {}, delay = 200 } = step;
+    
+    const result = await click(criteria, options);
+    if (!result.success) {
+      log(`clickSequence: Failed at step ${completedSteps + 1}`, 'warn');
+      return { success: false, completedSteps };
+    }
+    
+    completedSteps++;
+    await sleep(delay);
+  }
+  
+  return { success: true, completedSteps };
+}
+
+/**
+ * Perform hover over an element
+ * 
+ * @param {Object} criteria - Element search criteria
+ * @param {Object} [options] - Options
+ * @param {number} [options.duration=500] - How long to hover in ms
+ * @returns {Promise<{success: boolean, element: Object|null}>}
+ */
+async function hover(criteria, options = {}) {
+  const { duration = 500 } = options;
+  const { moveMouse } = require('../mouse');
+  
+  const element = await findElement(criteria);
+  if (!element) {
+    return { success: false, element: null };
+  }
+  
+  const bounds = element.bounds;
+  const x = bounds.x + bounds.width / 2;
+  const y = bounds.y + bounds.height / 2;
+  
+  await moveMouse(x, y);
+  await sleep(duration);
+  
+  return { success: true, element };
+}
+
+/**
+ * Wait for element and click
+ * Convenience wrapper combining wait + click
+ * 
+ * @param {Object} criteria - Element search criteria  
+ * @param {Object} [options] - Options
+ * @param {number} [options.timeout=5000] - Wait timeout
+ * @returns {Promise<{success: boolean, element: Object|null}>}
+ */
+async function waitAndClick(criteria, options = {}) {
+  const { timeout = 5000, ...clickOptions } = options;
+  return click(criteria, { ...clickOptions, waitTimeout: timeout });
+}
+
+/**
+ * Click an element then wait for another element to appear
+ * 
+ * @param {Object} clickCriteria - Element to click
+ * @param {Object} waitCriteria - Element to wait for
+ * @param {number} [timeout=10000] - Wait timeout
+ * @returns {Promise<{success: boolean, clickedElement?: Object, resultElement?: Object, error?: string}>}
+ */
+async function clickAndWaitFor(clickCriteria, waitCriteria, timeout = 10000) {
+  const clickResult = await click(clickCriteria);
+  if (!clickResult.success) {
+    return { success: false, error: `Click failed: ${clickResult.error || 'Element not found'}` };
+  }
+  
+  const waitResult = await waitForElement(waitCriteria, { timeout });
+  return {
+    success: !!waitResult,
+    clickedElement: clickResult.element,
+    resultElement: waitResult,
+    error: waitResult ? undefined : 'Wait timeout',
+  };
+}
+
+/**
+ * Select from a dropdown/combobox (alias for selectDropdownItem)
+ * 
+ * @param {Object} dropdownCriteria - Dropdown element criteria
+ * @param {string} optionText - Text of option to select
+ * @param {number} [timeout=5000] - Wait timeout for options
+ * @returns {Promise<{success: boolean, error?: string}>}
+ */
+async function selectFromDropdown(dropdownCriteria, optionText, timeout = 5000) {
+  // Click the dropdown to open it
+  const openResult = await click(dropdownCriteria);
+  if (!openResult.success) {
+    return { success: false, error: `Failed to open dropdown` };
+  }
+  
+  await sleep(200);
+  
+  // Find and click the option
+  const optionResult = await waitAndClick({ text: optionText }, { timeout });
+  if (!optionResult.success) {
+    // Try to close dropdown if option not found
+    const { sendKeys } = require('../keyboard');
+    await sendKeys('{ESC}');
+    return { success: false, error: `Option "${optionText}" not found` };
+  }
+  
+  return { success: true, selectedOption: optionText };
+}
+
+module.exports = {
+  fillField,
+  selectDropdownItem,
+  waitForWindow,
+  clickSequence,
+  hover,
+  waitAndClick,
+  clickAndWaitFor,
+  selectFromDropdown,
+};

package/src/main/ui-automation/interactions/index.js

@@ -0,0 +1,47 @@
+/**
+ * Interactions Module
+ * 
+ * @module ui-automation/interactions
+ */
+
+const { 
+  click, 
+  clickByText, 
+  clickByAutomationId, 
+  rightClick, 
+  doubleClick,
+  clickElement,
+  invokeElement,
+} = require('./element-click');
+
+const {
+  fillField,
+  selectDropdownItem,
+  waitForWindow,
+  clickSequence,
+  hover,
+  waitAndClick,
+  clickAndWaitFor,
+  selectFromDropdown,
+} = require('./high-level');
+
+module.exports = {
+  // Element clicks
+  click,
+  clickByText,
+  clickByAutomationId,
+  rightClick,
+  doubleClick,
+  clickElement,
+  invokeElement,
+  
+  // High-level interactions
+  fillField,
+  selectDropdownItem,
+  waitForWindow,
+  clickSequence,
+  hover,
+  waitAndClick,
+  clickAndWaitFor,
+  selectFromDropdown,
+};

package/src/main/ui-automation/keyboard/index.js

@@ -0,0 +1,15 @@
+/**
+ * Keyboard Module
+ * 
+ * @module ui-automation/keyboard
+ */
+
+const { typeText, sendKeys, keyDown, keyUp, VK } = require('./input');
+
+module.exports = {
+  typeText,
+  sendKeys,
+  keyDown,
+  keyUp,
+  VK,
+};