copilot-liku-cli 0.0.4 → 0.0.8

package/src/main/ui-automation/core/ui-provider.js

@@ -0,0 +1,99 @@
+const { spawn } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @typedef {Object} Bounds
+ * @property {number} x
+ * @property {number} y
+ * @property {number} width
+ * @property {number} height
+ */
+
+/**
+ * @typedef {Object} UIElement
+ * @property {string} id
+ * @property {string} name
+ * @property {string} role
+ * @property {Bounds} bounds
+ * @property {boolean} isClickable
+ * @property {boolean} isFocusable
+ * @property {UIElement[]} children
+ */
+
+class UIProvider {
+  constructor() {
+    const binDir = path.join(__dirname, '..', '..', '..', '..', 'bin');
+    const candidates = [
+      path.join(binDir, 'WindowsUIA.exe'),
+      path.join(binDir, 'windows-uia.exe')
+    ];
+    this.binaryPath = candidates.find(filePath => fs.existsSync(filePath)) || candidates[0];
+  }
+
+  /**
+   * Fetches the UI tree from the native binary.
+   * @returns {Promise<UIElement>}
+   */
+  async getUITree() {
+    return new Promise((resolve, reject) => {
+      if (!fs.existsSync(this.binaryPath)) {
+        return reject(new Error('UIAutomation binary not found. Build it with: powershell -ExecutionPolicy Bypass -File src/native/windows-uia/build.ps1'));
+      }
+
+      const child = spawn(this.binaryPath);
+      let output = '';
+      let errorOutput = '';
+
+      child.stdout.on('data', (data) => {
+        output += data.toString();
+      });
+
+      child.stderr.on('data', (data) => {
+        errorOutput += data.toString();
+      });
+
+      child.on('close', (code) => {
+        if (code !== 0) {
+          return reject(new Error(`Process exited with code ${code}: ${errorOutput}`));
+        }
+
+        try {
+          const parsed = JSON.parse(output);
+          const uiTree = this.parseNode(parsed);
+          resolve(uiTree);
+        } catch (err) {
+          reject(new Error(`Failed to parse JSON output: ${err.message}`));
+        }
+      });
+      
+      child.on('error', (err) => {
+        reject(new Error(`Failed to start subprocess: ${err.message}`));
+      });
+    });
+  }
+
+  /**
+   * Parses the OS-specific JSON node into a unified UIElement.
+   * @param {Object} node
+   * @returns {UIElement}
+   */
+  parseNode(node) {
+    return {
+      id: node.id || '',
+      name: node.name || '',
+      role: node.role || '',
+      bounds: {
+        x: node.bounds?.x || 0,
+        y: node.bounds?.y || 0,
+        width: node.bounds?.width || 0,
+        height: node.bounds?.height || 0
+      },
+      isClickable: !!node.isClickable,
+      isFocusable: !!node.isFocusable,
+      children: (node.children || []).map(child => this.parseNode(child))
+    };
+  }
+}
+
+module.exports = { UIProvider };

package/src/main/ui-automation/core/uia-host.js

@@ -0,0 +1,214 @@
+/**
+ * Persistent .NET UIA host — spawns WindowsUIA.exe once, communicates
+ * via newline-delimited JSON (JSONL) over stdin/stdout.
+ *
+ * Protocol:
+ *   stdin  → {"cmd":"elementFromPoint","x":500,"y":300}
+ *   stdout ← {"ok":true,"cmd":"elementFromPoint","element":{…}}
+ *
+ * Supported commands: getTree, elementFromPoint, exit.
+ */
+
+const { spawn } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const { EventEmitter } = require('events');
+
+const STARTUP_TIMEOUT_MS = 5000;
+const REQUEST_TIMEOUT_MS = 8000;
+
+class UIAHost extends EventEmitter {
+  constructor() {
+    super();
+    const binDir = path.join(__dirname, '..', '..', '..', '..', 'bin');
+    this._binaryPath = path.join(binDir, 'WindowsUIA.exe');
+    this._proc = null;
+    this._buffer = '';
+    this._pending = null; // { resolve, reject, timer }
+    this._alive = false;
+  }
+
+  /** Ensure the host process is running. Idempotent. */
+  async start() {
+    if (this._alive && this._proc && !this._proc.killed) return;
+
+    if (!fs.existsSync(this._binaryPath)) {
+      throw new Error(
+        `UIA host binary not found at ${this._binaryPath}. ` +
+        'Build with: powershell -ExecutionPolicy Bypass -File src/native/windows-uia-dotnet/build.ps1'
+      );
+    }
+
+    this._proc = spawn(this._binaryPath, [], {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      windowsHide: true
+    });
+
+    this._buffer = '';
+    this._alive = true;
+
+    this._proc.stdout.on('data', (chunk) => this._onData(chunk));
+    this._proc.stderr.on('data', (chunk) => {
+      this.emit('stderr', chunk.toString());
+    });
+    this._proc.on('exit', (code) => {
+      this._alive = false;
+      this._rejectPending(new Error(`UIA host exited with code ${code}`));
+      this.emit('exit', code);
+    });
+    this._proc.on('error', (err) => {
+      this._alive = false;
+      this._rejectPending(err);
+      this.emit('error', err);
+    });
+  }
+
+  /** Send a command and await the JSON response. */
+  async send(cmd) {
+    await this.start();
+
+    if (this._pending) {
+      throw new Error('UIAHost: concurrent request not supported (previous call still pending)');
+    }
+
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this._pending = null;
+        reject(new Error(`UIAHost: command "${cmd.cmd}" timed out after ${REQUEST_TIMEOUT_MS}ms`));
+      }, REQUEST_TIMEOUT_MS);
+
+      this._pending = { resolve, reject, timer };
+
+      const line = JSON.stringify(cmd) + '\n';
+      this._proc.stdin.write(line);
+    });
+  }
+
+  /** Convenience: elementFromPoint(x, y) → rich element payload */
+  async elementFromPoint(x, y) {
+    const resp = await this.send({ cmd: 'elementFromPoint', x, y });
+    if (!resp.ok) throw new Error(resp.error || 'elementFromPoint failed');
+    return resp.element;
+  }
+
+  /** Convenience: getTree() → foreground window tree */
+  async getTree() {
+    const resp = await this.send({ cmd: 'getTree' });
+    if (!resp.ok) throw new Error(resp.error || 'getTree failed');
+    return resp.tree;
+  }
+
+  /** Set value on element at (x,y) using ValuePattern. */
+  async setValue(x, y, value) {
+    const resp = await this.send({ cmd: 'setValue', x, y, value });
+    if (!resp.ok) throw new Error(resp.error || 'setValue failed');
+    return resp;
+  }
+
+  /** Scroll element at (x,y) using ScrollPattern. direction: up|down|left|right. amount: percent (0-100) or -1 for small increment. */
+  async scroll(x, y, direction = 'down', amount = -1) {
+    const resp = await this.send({ cmd: 'scroll', x, y, direction, amount });
+    if (!resp.ok) throw new Error(resp.error || 'scroll failed');
+    return resp;
+  }
+
+  /** Expand/collapse element at (x,y). action: expand|collapse|toggle. */
+  async expandCollapse(x, y, action = 'toggle') {
+    const resp = await this.send({ cmd: 'expandCollapse', x, y, action });
+    if (!resp.ok) throw new Error(resp.error || 'expandCollapse failed');
+    return resp;
+  }
+
+  /** Get text from element at (x,y) using TextPattern → ValuePattern → Name fallback. */
+  async getText(x, y) {
+    const resp = await this.send({ cmd: 'getText', x, y });
+    if (!resp.ok) throw new Error(resp.error || 'getText failed');
+    return resp;
+  }
+
+  /** Subscribe to UIA events (focus, structure, property). Returns initial snapshot. */
+  async subscribeEvents() {
+    const resp = await this.send({ cmd: 'subscribeEvents' });
+    if (!resp.ok) throw new Error(resp.error || 'subscribeEvents failed');
+    return resp;
+  }
+
+  /** Unsubscribe from all UIA events. */
+  async unsubscribeEvents() {
+    const resp = await this.send({ cmd: 'unsubscribeEvents' });
+    if (!resp.ok) throw new Error(resp.error || 'unsubscribeEvents failed');
+    return resp;
+  }
+
+  /** Gracefully shut down the host process. */
+  async stop() {
+    if (!this._alive || !this._proc) return;
+    try {
+      await this.send({ cmd: 'exit' });
+    } catch { /* ignore */ }
+    this._alive = false;
+    if (this._proc && !this._proc.killed) {
+      this._proc.kill();
+    }
+    this._proc = null;
+  }
+
+  get isAlive() {
+    return this._alive;
+  }
+
+  // ── internal ─────────────────────────────────────────────────────────
+
+  _onData(chunk) {
+    this._buffer += chunk.toString();
+    let nl;
+    while ((nl = this._buffer.indexOf('\n')) !== -1) {
+      const line = this._buffer.slice(0, nl).trim();
+      this._buffer = this._buffer.slice(nl + 1);
+      if (!line) continue;
+      try {
+        const json = JSON.parse(line);
+        // Phase 4: route unsolicited event messages before pending resolution
+        if (json.type === 'event') {
+          this.emit('uia-event', json);
+          continue;
+        }
+        this._resolvePending(json);
+      } catch (e) {
+        this.emit('parseError', line, e);
+      }
+    }
+  }
+
+  _resolvePending(json) {
+    if (!this._pending) return;
+    const { resolve, timer } = this._pending;
+    clearTimeout(timer);
+    this._pending = null;
+    resolve(json);
+  }
+
+  _rejectPending(err) {
+    if (!this._pending) return;
+    const { reject, timer } = this._pending;
+    clearTimeout(timer);
+    this._pending = null;
+    reject(err);
+  }
+}
+
+// Singleton for shared use
+let _shared = null;
+
+/**
+ * Get or create the shared UIAHost instance.
+ * @returns {UIAHost}
+ */
+function getSharedUIAHost() {
+  if (!_shared) {
+    _shared = new UIAHost();
+  }
+  return _shared;
+}
+
+module.exports = { UIAHost, getSharedUIAHost };

package/src/main/ui-automation/index.js

@@ -28,6 +28,8 @@ const { CONFIG, CONTROL_TYPES } = require('./config');
 
 // Core utilities
 const { sleep, debug, log, executePowerShellScript } = require('./core');
+const { UIProvider } = require('./core/ui-provider');
+const { UIAHost, getSharedUIAHost } = require('./core/uia-host');
 
 // Element operations
 const { 
@@ -64,7 +66,10 @@ const {
 const {
   getActiveWindow,
   findWindows,
+  resolveWindowTarget,
   focusWindow,
+  bringWindowToFront,
+  sendWindowToBack,
   minimizeWindow,
   maximizeWindow,
   restoreWindow,
@@ -87,6 +92,15 @@ const {
   waitAndClick,
   clickAndWaitFor,
   selectFromDropdown,
+  // Pattern-based interactions (Phase 3)
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
 } = require('./interactions');
 
 // Screenshot
@@ -106,6 +120,9 @@ module.exports = {
   debug,
   log,
   executePowerShellScript,
+  UIProvider,
+  UIAHost,
+  getSharedUIAHost,
   
   // Element operations
   findElements,
@@ -135,7 +152,10 @@ module.exports = {
   // Window operations
   getActiveWindow,
   findWindows,
+  resolveWindowTarget,
   focusWindow,
+  bringWindowToFront,
+  sendWindowToBack,
   minimizeWindow,
   maximizeWindow,
   restoreWindow,
@@ -157,6 +177,16 @@ module.exports = {
   clickAndWaitFor,
   selectFromDropdown,
   
+  // Pattern-based interactions (Phase 3)
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
+  
   // Screenshot
   screenshot,
   screenshotActiveWindow,

package/src/main/ui-automation/interactions/element-click.js

@@ -50,10 +50,10 @@ async function click(criteria, options = {}) {
     return { success: false, element: null, error: findResult?.error || 'Element not found' };
   }
   
-  // Calculate center point
+  // Calculate click point — prefer UIA clickPoint over bounds-center
   const bounds = element.bounds;
-  const x = bounds.x + bounds.width / 2;
-  const y = bounds.y + bounds.height / 2;
+  const x = element.clickPoint?.x ?? (bounds.x + bounds.width / 2);
+  const y = element.clickPoint?.y ?? (bounds.y + bounds.height / 2);
   
   // Focus window if needed
   if (focusWindow && element.windowHwnd) {
@@ -132,11 +132,11 @@ async function clickElement(element, options = {}) {
   }
   
   const bounds = element.bounds;
-  const centerX = bounds.x + bounds.width / 2;
-  const centerY = bounds.y + bounds.height / 2;
+  const centerX = element.clickPoint?.x ?? (bounds.x + bounds.width / 2);
+  const centerY = element.clickPoint?.y ?? (bounds.y + bounds.height / 2);
   
   // Strategy 1: Try Invoke pattern for buttons
-  if (useInvoke && element.patterns?.includes('InvokePatternIdentifiers.Pattern')) {
+  if (useInvoke && (element.patterns?.includes('InvokePatternIdentifiers.Pattern') || element.patterns?.includes('Invoke'))) {
     log(`Attempting Invoke pattern for "${element.name}"`);
     const invokeResult = await invokeElement(element);
     if (invokeResult.success) {

package/src/main/ui-automation/interactions/high-level.js

@@ -7,6 +7,7 @@
 
 const { findElement, findElements, waitForElement } = require('../elements');
 const { click, clickByText } = require('./element-click');
+const { setElementValue, expandElement } = require('./pattern-actions');
 const { typeText, sendKeys } = require('../keyboard');
 const { focusWindow, findWindows } = require('../window');
 const { log, sleep } = require('../core/helpers');
@@ -21,9 +22,18 @@ const { log, sleep } = require('../core/helpers');
  * @returns {Promise<{success: boolean}>}
  */
 async function fillField(criteria, text, options = {}) {
-  const { clear = true } = options;
+  const { clear = true, preferPattern = true } = options;
+  
+  // Strategy 1: Try ValuePattern (fast, no focus/click needed)
+  if (preferPattern) {
+    const patternResult = await setElementValue(criteria, text);
+    if (patternResult.success) {
+      log(`fillField: ValuePattern succeeded for "${text.slice(0, 30)}"`);
+      return { success: true, method: 'ValuePattern' };
+    }
+  }
   
-  // Click the field
+  // Strategy 2: Click + type (fallback)
   const clickResult = await click(criteria);
   if (!clickResult.success) {
     return { success: false };
@@ -39,7 +49,7 @@ async function fillField(criteria, text, options = {}) {
   
   // Type text
   const typeResult = await typeText(text);
-  return { success: typeResult.success };
+  return { success: typeResult.success, method: 'sendKeys' };
 }
 
 /**
@@ -52,9 +62,21 @@ async function fillField(criteria, text, options = {}) {
  * @returns {Promise<{success: boolean}>}
  */
 async function selectDropdownItem(dropdownCriteria, itemCriteria, options = {}) {
-  const { itemWait = 1000 } = options;
+  const { itemWait = 1000, preferPattern = true } = options;
+  
+  // Strategy 1: Try ExpandCollapsePattern to open
+  if (preferPattern) {
+    const expandResult = await expandElement(dropdownCriteria);
+    if (expandResult.success) {
+      log(`selectDropdownItem: ExpandCollapsePattern expanded (${expandResult.stateBefore} → ${expandResult.stateAfter})`);
+      await sleep(itemWait);
+      const itemQuery = typeof itemCriteria === 'string' ? { text: itemCriteria } : itemCriteria;
+      const itemResult = await click(itemQuery);
+      return { success: itemResult.success, method: 'ExpandCollapsePattern' };
+    }
+  }
   
-  // Click dropdown to open
+  // Strategy 2: Click to open (fallback)
   const openResult = await click(dropdownCriteria);
   if (!openResult.success) {
     log('selectDropdownItem: Failed to open dropdown', 'warn');
@@ -69,7 +91,7 @@ async function selectDropdownItem(dropdownCriteria, itemCriteria, options = {})
     : itemCriteria;
     
   const itemResult = await click(itemQuery);
-  return { success: itemResult.success };
+  return { success: itemResult.success, method: 'click' };
 }
 
 /**

package/src/main/ui-automation/interactions/index.js

@@ -25,6 +25,17 @@ const {
   selectFromDropdown,
 } = require('./high-level');
 
+const {
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
+} = require('./pattern-actions');
+
 module.exports = {
   // Element clicks
   click,
@@ -44,4 +55,14 @@ module.exports = {
   waitAndClick,
   clickAndWaitFor,
   selectFromDropdown,
+  
+  // Pattern-based interactions (Phase 3)
+  normalizePatternName,
+  hasPattern,
+  setElementValue,
+  scrollElement,
+  expandElement,
+  collapseElement,
+  toggleExpandCollapse,
+  getElementText,
 };