@ai-agent-forge/plugin-desktop-windows-input 0.88.1 → 0.88.3

package/dist/windows-uia-adapter.js

@@ -19,6 +19,12 @@ const SUPPORTED_ACTIONS = [
     "key",
     "scroll",
 ];
+/** Contract limit: a delivered screenshot must stay at or below 4 MiB decoded. */
+const DESKTOP_SCREENSHOT_MAX_DECODED_BYTES = 4 * 1024 * 1024;
+/** Spawn buffer headroom above the contract limit so the limit is enforced here, not by maxBuffer. */
+const SCREENSHOT_SPAWN_MAX_BUFFER_BYTES = 16 * 1024 * 1024;
+/** Screens wider than this are scaled down proportionally before PNG encoding. */
+const DESKTOP_SCREENSHOT_MAX_WIDTH_PX = 1280;
 const error = (code, message, causeCode) => ({
     version: 1,
     code,
@@ -38,6 +44,7 @@ const rectangle = (raw) => ({
     width: Number.isFinite(raw.width) ? Number(raw.width) : 0,
     height: Number.isFinite(raw.height) ? Number(raw.height) : 0,
 });
+const screenshotDimension = (value) => typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) && value > 0 ? value : 0;
 const QUERY_SCRIPT = `
 try {
   Add-Type -AssemblyName UIAutomationClient -ErrorAction Stop
@@ -226,29 +233,48 @@ public static class AgentForgeNativeText {
       } catch { throw 'Target text update failed' }
     }
   } elseif ($payload.action.kind -eq 'key') {
-    $keyText = [string]$payload.action.key
-    $hasSendKeysSyntax = $keyText -match '[+%^{}]'
-    if (-not $hasSendKeysSyntax -and $controlHandle -gt 0) {
-      # Plain text with a real control handle: deliver synchronously through
-      # SendMessage WM_CHAR. Queued input (SendInput / WScript SendKeys) never
-      # reaches GUI windows in non-interactive or remote desktop sessions, while
-      # synchronous SendMessage does — the same path that type (WM_SETTEXT) and
-      # click (BM_CLICK) already use. Keyboard focus is not required.
+    # Real keyboard-queue input: keybd_event posts to the same queue as the
+    # physical keyboard. The native-file-picker flow depends on this — the
+    # dialog is focused but invisible to UIA and not addressable by window
+    # messages sent to the owner window. Named keys and Ctrl/Alt/Shift
+    # combinations go through virtual-key codes; anything else must be a
+    # single character.
+    try {
       Add-Type @'
 using System;
 using System.Runtime.InteropServices;
-public static class AgentForgeNativeKey {
-  [DllImport("user32.dll", SetLastError=true)] public static extern IntPtr SendMessage(IntPtr hWnd, uint Msg, IntPtr wParam, IntPtr lParam);
+public static class AgentForgeNativeKeyQueue {
+  [DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
 }
 '@
-      foreach ($ch in $keyText.ToCharArray()) {
-        [AgentForgeNativeKey]::SendMessage([IntPtr]$controlHandle, 0x0102, [IntPtr][char]$ch, [IntPtr]::Zero) | Out-Null
-      }
-    } else {
-      if ($payload.target.controlId) { $target.SetFocus() }
-      $wshell = New-Object -ComObject WScript.Shell
-      $wshell.SendKeys($keyText)
-    }
+    } catch {}
+    $vkMap = @{
+      'enter'=0x0D; 'return'=0x0D; 'tab'=0x09; 'escape'=0x1B; 'esc'=0x1B;
+      'backspace'=0x08; 'delete'=0x2E; 'space'=0x20;
+      'up'=0x26; 'down'=0x28; 'left'=0x25; 'right'=0x27;
+      'arrowup'=0x26; 'arrowdown'=0x28; 'arrowleft'=0x25; 'arrowright'=0x27;
+      'home'=0x24; 'end'=0x23; 'pageup'=0x21; 'pagedown'=0x22; 'insert'=0x2D
+    }
+    $modMap = @{ 'ctrl'=0x11; 'control'=0x11; 'alt'=0x12; 'shift'=0x10; 'meta'=0x5B; 'win'=0x5B; 'cmd'=0x5B }
+    $keyText = ([string]$payload.action.key).Trim()
+    $mods = @()
+    $base = $null
+    foreach ($p in ($keyText -split '+')) {
+      $lp = $p.ToLower()
+      if ($modMap.ContainsKey($lp)) { $mods += $modMap[$lp] } else { $base = $lp }
+    }
+    $vk = $null
+    if ($null -ne $base -and $vkMap.ContainsKey($base)) { $vk = $vkMap[$base] }
+    elseif ($null -ne $base -and $base.Length -eq 1) {
+      $upper = [char]::ToUpper($base[0])
+      $code = [int]$upper
+      if (($code -ge 0x20 -and $code -le 0x5A) -or ($code -ge 0x30 -and $code -le 0x39)) { $vk = $code }
+    }
+    if ($null -eq $vk) { throw ('Unsupported key: ' + $keyText) }
+    foreach ($m in $mods) { [AgentForgeNativeKeyQueue]::keybd_event([byte]$m, 0, 0, [UIntPtr]::Zero) }
+    [AgentForgeNativeKeyQueue]::keybd_event([byte]$vk, 0, 0, [UIntPtr]::Zero)
+    [AgentForgeNativeKeyQueue]::keybd_event([byte]$vk, 0, 2, [UIntPtr]::Zero)
+    for ($i = $mods.Count - 1; $i -ge 0; $i--) { [AgentForgeNativeKeyQueue]::keybd_event([byte]$mods[$i], 0, 2, [UIntPtr]::Zero) }
   } elseif ($payload.action.kind -eq 'scroll') {
     Add-Type @'
 using System;
@@ -282,6 +308,128 @@ public static class AgentForgeNativeScroll {
   $reason = if ($message -match 'ValuePattern|LegacyIAccessiblePattern') { 'value_pattern_unavailable' } elseif ($message -match 'InvokePattern') { 'invoke_pattern_unavailable' } elseif ($message -match 'SetFocus') { 'focus_failed' } else { 'uia_action_exception' }
   [pscustomobject]@{ ok=$false; code=$code; reason=$reason } | ConvertTo-Json -Compress
 }`;
+/**
+ * Foreground (no-target) type/key: real keyboard-queue input for surfaces UIA
+ * cannot see — the native file dialog's focused name box being the canonical
+ * case. Named keys go through virtual-key codes with modifier support; other
+ * text is typed as VK_PACKET unicode events.
+ */
+const FOREGROUND_INPUT_SCRIPT = `
+try {
+  $payload = $env:AGENT_FORGE_UIA_INPUT | ConvertFrom-Json
+  try {
+    Add-Type @'
+using System;
+using System.Runtime.InteropServices;
+public static class AgentForgeForegroundKeyQueue {
+  [DllImport("user32.dll")] public static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, UIntPtr dwExtraInfo);
+}
+'@
+  } catch {}
+  $vkMap = @{
+    'enter'=0x0D; 'return'=0x0D; 'tab'=0x09; 'escape'=0x1B; 'esc'=0x1B;
+    'backspace'=0x08; 'delete'=0x2E; 'space'=0x20;
+    'up'=0x26; 'down'=0x28; 'left'=0x25; 'right'=0x27;
+    'arrowup'=0x26; 'arrowdown'=0x28; 'arrowleft'=0x25; 'arrowright'=0x27;
+    'home'=0x24; 'end'=0x23; 'pageup'=0x21; 'pagedown'=0x22; 'insert'=0x2D
+  }
+  $modMap = @{ 'ctrl'=0x11; 'control'=0x11; 'alt'=0x12; 'shift'=0x10; 'meta'=0x5B; 'win'=0x5B; 'cmd'=0x5B }
+  if ($payload.action.kind -eq 'type') {
+    foreach ($ch in ([string]$payload.action.text).ToCharArray()) {
+      [AgentForgeForegroundKeyQueue]::keybd_event(0xE7, [byte][int]$ch, 0, [UIntPtr]::Zero)
+      [AgentForgeForegroundKeyQueue]::keybd_event(0xE7, [byte][int]$ch, 2, [UIntPtr]::Zero)
+    }
+  } elseif ($payload.action.kind -eq 'key') {
+    $keyText = ([string]$payload.action.key).Trim()
+    $mods = @()
+    $base = $null
+    foreach ($p in ($keyText -split '\\+')) {
+      $lp = $p.ToLower()
+      if ($modMap.ContainsKey($lp)) { $mods += $modMap[$lp] } else { $base = $lp }
+    }
+    $vk = $null
+    if ($null -ne $base -and $vkMap.ContainsKey($base)) { $vk = $vkMap[$base] }
+    elseif ($null -ne $base -and $base.Length -eq 1) {
+      $upper = [char]::ToUpper($base[0])
+      $code = [int]$upper
+      if (($code -ge 0x20 -and $code -le 0x5A) -or ($code -ge 0x30 -and $code -le 0x39)) { $vk = $code }
+    }
+    if ($null -eq $vk) { throw ('Unsupported key: ' + $keyText) }
+    foreach ($m in $mods) { [AgentForgeForegroundKeyQueue]::keybd_event([byte]$m, 0, 0, [UIntPtr]::Zero) }
+    [AgentForgeForegroundKeyQueue]::keybd_event([byte]$vk, 0, 0, [UIntPtr]::Zero)
+    [AgentForgeForegroundKeyQueue]::keybd_event([byte]$vk, 0, 2, [UIntPtr]::Zero)
+    for ($i = $mods.Count - 1; $i -ge 0; $i--) { [AgentForgeForegroundKeyQueue]::keybd_event([byte]$mods[$i], 0, 2, [UIntPtr]::Zero) }
+  } else { throw 'Foreground input supports only type and key' }
+  [pscustomobject]@{ ok=$true } | ConvertTo-Json -Compress
+} catch {
+  [pscustomobject]@{ ok=$false; code='action_failed'; reason=[string]$_.Exception.Message } | ConvertTo-Json -Compress
+}`;
+const COORDINATE_CLICK_SCRIPT = `
+try {
+  $payload = $env:AGENT_FORGE_UIA_INPUT | ConvertFrom-Json
+  $x = [int]$payload.target.x
+  $y = [int]$payload.target.y
+  Add-Type @'
+using System;
+using System.Runtime.InteropServices;
+public static class AgentForgeCoordinateClick {
+  [DllImport("user32.dll", SetLastError=true)]
+  public static extern bool SetCursorPos(int X, int Y);
+  [DllImport("user32.dll")]
+  public static extern void mouse_event(uint dwFlags, int dx, int dy, int dwData, UIntPtr dwExtraInfo);
+}
+'@
+  if (-not [AgentForgeCoordinateClick]::SetCursorPos($x, $y)) { throw 'Coordinate click could not move the cursor' }
+  [AgentForgeCoordinateClick]::mouse_event(0x0002, 0, 0, 0, [UIntPtr]::Zero)
+  [AgentForgeCoordinateClick]::mouse_event(0x0004, 0, 0, 0, [UIntPtr]::Zero)
+  [pscustomobject]@{ ok=$true } | ConvertTo-Json -Compress
+} catch {
+  [pscustomobject]@{ ok=$false; reason='uia_action_exception' } | ConvertTo-Json -Compress
+}`;
+const SCREENSHOT_SCRIPT = `
+try {
+  Add-Type @'
+using System;
+using System.Runtime.InteropServices;
+public static class AgentForgeScreenDpi {
+  [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();
+}
+'@
+  [void][AgentForgeScreenDpi]::SetProcessDPIAware()
+  Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop
+  Add-Type -AssemblyName System.Drawing -ErrorAction Stop
+  $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+  $originalWidth = [int]$bounds.Width
+  $originalHeight = [int]$bounds.Height
+  if ($originalWidth -le 0 -or $originalHeight -le 0) { throw 'Primary display bounds are unavailable' }
+  $width = if ($originalWidth -gt ${DESKTOP_SCREENSHOT_MAX_WIDTH_PX}) { ${DESKTOP_SCREENSHOT_MAX_WIDTH_PX} } else { $originalWidth }
+  $height = [int][math]::Round([double]$originalHeight * $width / $originalWidth)
+  if ($height -lt 1) { $height = 1 }
+  $source = New-Object System.Drawing.Bitmap($originalWidth, $originalHeight)
+  $target = $source
+  try {
+    $graphics = [System.Drawing.Graphics]::FromImage($source)
+    try { $graphics.CopyFromScreen($bounds.X, $bounds.Y, 0, 0, [System.Drawing.Size]::new($originalWidth, $originalHeight)) } finally { $graphics.Dispose() }
+    if ($width -ne $originalWidth -or $height -ne $originalHeight) {
+      $target = New-Object System.Drawing.Bitmap($width, $height)
+      $resized = [System.Drawing.Graphics]::FromImage($target)
+      try {
+        $resized.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic
+        $resized.DrawImage($source, 0, 0, $width, $height)
+      } finally { $resized.Dispose() }
+    }
+    $stream = New-Object System.IO.MemoryStream
+    try {
+      $target.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png)
+      [pscustomobject]@{ ok=$true; data=[Convert]::ToBase64String($stream.ToArray()); width=$width; height=$height; originalWidth=$originalWidth; originalHeight=$originalHeight } | ConvertTo-Json -Compress -Depth 4
+    } finally { $stream.Dispose() }
+  } finally {
+    if (-not [object]::ReferenceEquals($target, $source)) { $target.Dispose() }
+    $source.Dispose()
+  }
+} catch {
+  [pscustomobject]@{ ok=$false; reason='screenshot_exception' } | ConvertTo-Json -Compress -Depth 4
+}`;
 const parseJson = (stdout) => {
     try {
         const parsed = JSON.parse(stdout.trim());
@@ -390,23 +538,49 @@ export class WindowsUiAutomationDriver {
             };
         if (!SUPPORTED_ACTIONS.includes(request.action.kind))
             return unsupported(request.operationId, "The selected Windows UI Automation adapter does not support this action", "action-unsupported");
-        if (request.action.target === undefined)
+        const postconditions = request.postconditions ?? [];
+        if (postconditions.length > 0 && request.reobserveAfterAction === false)
+            return this.failed(request.operationId, "reobserve_required", "Postconditions require an action re-observation");
+        // Coordinate clicks address raw screen pixels; they bypass observed target
+        // identity entirely and are click-only by contract.
+        if (request.action.target !== undefined && request.action.target.kind === "coordinates") {
+            if (request.action.kind !== "click")
+                return unsupported(request.operationId, "The selected Windows UI Automation adapter does not support this action", "action-unsupported");
+            const raw = this.run(COORDINATE_CLICK_SCRIPT, {
+                action: request.action,
+                target: request.action.target,
+            });
+            return this.completeAction(request, postconditions, raw);
+        }
+        // No observed target: type/key go to the real keyboard queue addressed at
+        // the current focus — the only path that reaches UIA-invisible surfaces
+        // like the native file dialog.
+        if (request.action.target === undefined) {
+            if (request.action.kind === "type" && typeof request.action.text === "string") {
+                const raw = this.run(FOREGROUND_INPUT_SCRIPT, { action: request.action });
+                return this.completeAction(request, postconditions, raw);
+            }
+            if (request.action.kind === "key" && typeof request.action.key === "string") {
+                const raw = this.run(FOREGROUND_INPUT_SCRIPT, { action: request.action });
+                return this.completeAction(request, postconditions, raw);
+            }
             return this.failed(request.operationId, "target_required", "This desktop action requires an observed target");
+        }
         if (request.action.kind === "type" && typeof request.action.text !== "string")
             return this.failed(request.operationId, "text_required", "Type action requires text");
         if (request.action.kind === "key" && typeof request.action.key !== "string")
             return this.failed(request.operationId, "key_required", "Key action requires a key sequence");
         if (request.action.kind === "scroll" && typeof request.action.text !== "string")
             return this.failed(request.operationId, "direction_required", "Scroll action requires an up/down direction");
-        const postconditions = request.postconditions ?? [];
-        if (postconditions.length > 0 && request.reobserveAfterAction === false)
-            return this.failed(request.operationId, "reobserve_required", "Postconditions require an action re-observation");
         const target = this.resolveTarget(request.sessionId, request.action.target);
         if (target === undefined)
             return this.failed(request.operationId, this.observedSessions.has(request.sessionId) ? "stale_target" : "target_not_found", this.observedSessions.has(request.sessionId)
                 ? "Target identity is stale; reobserve the desktop before retrying"
                 : "Target identity was not observed by UI Automation");
         const raw = this.run(ACTION_SCRIPT, { action: request.action, target: target ?? {} });
+        return this.completeAction(request, postconditions, raw);
+    }
+    completeAction(request, postconditions, raw) {
         if (raw === undefined)
             return unsupported(request.operationId, "Windows UI Automation action failed");
         if (raw.ok !== true) {
@@ -464,7 +638,63 @@ export class WindowsUiAutomationDriver {
         this.invalidateObservation(sessionId);
         return stopped;
     }
-    run(script, payload) {
+    /** Capture the primary display through GDI; the PNG is scaled to a 1280px width cap. */
+    screenshot() {
+        if (this.platform !== "win32")
+            return {
+                version: 1,
+                source: "windows-gdi",
+                status: "unsupported",
+                error: error("platform_unavailable", "Windows GDI screen capture requires win32", "platform-unavailable"),
+            };
+        const raw = this.run(SCREENSHOT_SCRIPT, undefined, SCREENSHOT_SPAWN_MAX_BUFFER_BYTES);
+        if (raw === undefined)
+            return {
+                version: 1,
+                source: "windows-gdi",
+                status: "failed",
+                error: error("screenshot_failed", "Windows GDI screen capture failed", "screenshot-capture-unavailable"),
+            };
+        if (raw.ok !== true)
+            return {
+                version: 1,
+                source: "windows-gdi",
+                status: "failed",
+                error: error("screenshot_failed", "Windows GDI screen capture failed", "screenshot-exception"),
+            };
+        const data = typeof raw.data === "string" ? raw.data : "";
+        const dimensions = {
+            width: screenshotDimension(raw.width),
+            height: screenshotDimension(raw.height),
+            originalWidth: screenshotDimension(raw.originalWidth),
+            originalHeight: screenshotDimension(raw.originalHeight),
+        };
+        if (Math.floor((data.length * 3) / 4) > DESKTOP_SCREENSHOT_MAX_DECODED_BYTES)
+            return {
+                version: 1,
+                source: "windows-gdi",
+                status: "failed",
+                error: error("screenshot-too-large", "The captured screenshot exceeds the 4 MiB decoded-size limit", "screenshot-too-large"),
+            };
+        if (data.length === 0 || Object.values(dimensions).some((value) => value <= 0))
+            return {
+                version: 1,
+                source: "windows-gdi",
+                status: "failed",
+                error: error("screenshot_failed", "Windows GDI screen capture returned an invalid payload", "screenshot-invalid-payload"),
+            };
+        return {
+            version: 1,
+            source: "windows-gdi",
+            status: "succeeded",
+            mimeType: "image/png",
+            data,
+            ...dimensions,
+            scale: dimensions.originalWidth / dimensions.width,
+            capturedAt: new Date().toISOString(),
+        };
+    }
+    run(script, payload, maxBuffer) {
         if (this.platform !== "win32")
             return undefined;
         const env = {};
@@ -475,7 +705,13 @@ export class WindowsUiAutomationDriver {
         }
         if (payload !== undefined)
             env.AGENT_FORGE_UIA_INPUT = JSON.stringify(payload);
-        const result = this.spawn(this.command, ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", script], { encoding: "utf8", timeout: this.timeoutMs, windowsHide: true, env });
+        const result = this.spawn(this.command, ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", script], {
+            encoding: "utf8",
+            timeout: this.timeoutMs,
+            windowsHide: true,
+            env,
+            ...(maxBuffer === undefined ? {} : { maxBuffer }),
+        });
         if (result.error !== undefined || result.status !== 0)
             return undefined;
         return parseJson(result.stdout);
@@ -563,6 +799,8 @@ export class WindowsUiAutomationDriver {
         return observation;
     }
     resolveTarget(sessionId, target) {
+        if (target.kind === "coordinates")
+            return undefined;
         const identity = target.kind === "control" ? target.controlId : target.windowId;
         const record = this.targets.get(this.targetKey(sessionId, target.kind, identity));
         if (record === undefined ||
@@ -612,6 +850,8 @@ export class WindowsUiAutomationDriver {
                 return false;
             return window.title === condition.expected;
         }
+        if (target.kind !== "control")
+            return false;
         const control = observation.controls.find((item) => {
             if (item.controlId !== target.controlId)
                 return false;
@@ -674,6 +914,18 @@ export class DesktopWindowsAdapter {
     stop(sessionId) {
         return this.driver.stop(sessionId);
     }
+    /** Delegates to the driver when it implements DesktopScreenshotDriverV1; structured unsupported otherwise. */
+    screenshot() {
+        const driver = this.driver;
+        if (typeof driver.screenshot !== "function")
+            return {
+                version: 1,
+                source: "windows-gdi",
+                status: "unsupported",
+                error: error("driver_unavailable", "The selected desktop driver does not support screenshots", "screenshot-driver-unavailable"),
+            };
+        return driver.screenshot();
+    }
     replay(request) {
         const mismatchedOperationId = findDesktopReplaySessionMismatch(request);
         if (mismatchedOperationId !== undefined)