@midscene/computer 1.13.0 → 1.13.1-beta-20260921032526.0

package/dist/es/cli.mjs

@@ -425,6 +425,20 @@ $midsceneSetDpiMethod.SetImplementationFlags(
   $midsceneSetDpiMethod.GetMethodImplementationFlags() -bor
   [System.Reflection.MethodImplAttributes]::PreserveSig
 )
+$midsceneGetDpiForSystemMethod = $midsceneType.DefinePInvokeMethod(
+  'GetDpiForSystem',
+  'user32.dll',
+  $midsceneMethodAttributes,
+  [System.Reflection.CallingConventions]::Standard,
+  [uint32],
+  [System.Type[]]@(),
+  [System.Runtime.InteropServices.CallingConvention]::Winapi,
+  [System.Runtime.InteropServices.CharSet]::None
+)
+$midsceneGetDpiForSystemMethod.SetImplementationFlags(
+  $midsceneGetDpiForSystemMethod.GetMethodImplementationFlags() -bor
+  [System.Reflection.MethodImplAttributes]::PreserveSig
+)
 $midsceneGetForegroundWindowMethod = $midsceneType.DefinePInvokeMethod(
   'GetForegroundWindow',
   'user32.dll',
@@ -461,12 +475,8 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
   throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
 }
 `.trim();
-function runWindowsPhysicalPixelPowershell(script) {
-    const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
-$ErrorActionPreference = 'Stop'
-${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
-${script}`;
-    const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
+function runWindowsPowershellScript(script) {
+    const encoded = Buffer.from(script, 'utf16le').toString('base64');
     return execFileSync('powershell.exe', [
         '-NoProfile',
         '-NonInteractive',
@@ -479,6 +489,130 @@ ${script}`;
         windowsHide: true
     });
 }
+function runWindowsPowershellCommand(script) {
+    return execFileSync('powershell.exe', [
+        '-NoProfile',
+        '-Command',
+        script
+    ], {
+        encoding: 'utf8',
+        timeout: POWERSHELL_TIMEOUT_MS,
+        maxBuffer: POWERSHELL_MAX_BUFFER,
+        windowsHide: true
+    });
+}
+function runWindowsPowershell(script) {
+    return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
+$ErrorActionPreference = 'Stop'
+${script}`);
+}
+function runWindowsPhysicalPixelPowershell(script) {
+    const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
+$ErrorActionPreference = 'Stop'
+${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
+${script}`;
+    return runWindowsPowershellScript(physicalPixelScript);
+}
+class WindowsDisplayEnumerationEmptyError extends Error {
+}
+function isFiniteNumber(value) {
+    return 'number' == typeof value && Number.isFinite(value);
+}
+function isWindowsDisplayGeometry(value) {
+    if (!value || 'object' != typeof value) return false;
+    const candidate = value;
+    const bounds = candidate.bounds;
+    return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
+}
+function readWindowsDisplayGeometriesWith(runPowershell, context) {
+    const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
+    const script = `
+Add-Type -AssemblyName System.Windows.Forms
+$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
+  $bounds = $_.Bounds
+  [PSCustomObject]@{
+    id = $_.DeviceName
+    name = $_.DeviceName
+    primary = $_.Primary
+    bounds = [PSCustomObject]@{
+      x = $bounds.X
+      y = $bounds.Y
+      width = $bounds.Width
+      height = $bounds.Height
+    }
+  }
+})
+${serializeScreens}
+`.trim();
+    const output = runPowershell(script).trim();
+    if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
+    const parsed = JSON.parse(output);
+    if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
+    const displays = parsed.filter(isWindowsDisplayGeometry);
+    if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
+    if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
+    return displays;
+}
+function readWindowsDisplayGeometries() {
+    return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
+}
+function discoverWindowsDisplays(runners = {
+    physical: runWindowsPhysicalPixelPowershell,
+    legacy: runWindowsPowershell
+}) {
+    try {
+        return {
+            geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
+            coordinateMode: 'physical'
+        };
+    } catch (error) {
+        if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
+        const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
+        return {
+            geometries,
+            coordinateMode: 'legacy'
+        };
+    }
+}
+function readPngDimensions(dataUri) {
+    const prefix = 'data:image/png;base64,';
+    if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
+    const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
+    const pngSignature = Buffer.from([
+        137,
+        80,
+        78,
+        71,
+        13,
+        10,
+        26,
+        10
+    ]);
+    if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
+    const width = buffer.readUInt32BE(16);
+    const height = buffer.readUInt32BE(20);
+    if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
+    return {
+        width,
+        height
+    };
+}
+function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
+    if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
+    if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
+    const screenshotSize = readPngDimensions(screenshotBase64);
+    const displaySize = {
+        width: Math.round(geometry.bounds.width),
+        height: Math.round(geometry.bounds.height)
+    };
+    const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
+    if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
+}
+function resolveWindowsDisplayGeometryFromList(displayId, displays) {
+    if (!displays.length) return;
+    if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
+    return displays.find((display)=>display.id === displayId);
+}
 function windows_pointer_define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -796,28 +930,6 @@ const EDGE_SCROLL_SPEC = {
 function escapePowershellSingleQuoted(value) {
     return value.replace(/'/g, "''");
 }
-function readWindowsDisplayGeometries() {
-    const script = `
-Add-Type -AssemblyName System.Windows.Forms
-$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
-  $b = $_.Bounds
-  [PSCustomObject]@{
-    id = $_.DeviceName
-    name = $_.DeviceName
-    primary = $_.Primary
-    bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
-  }
-}
-ConvertTo-Json @($s) -Compress
-`.trim();
-    const output = runWindowsPhysicalPixelPowershell(script).trim();
-    if (!output) throw new Error('Windows display enumeration returned no data');
-    const parsed = JSON.parse(output);
-    if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
-    const displays = parsed.filter(isWindowsDisplayGeometry);
-    if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
-    return displays;
-}
 function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
     return geometries.map((display)=>({
             id: display.id,
@@ -921,23 +1033,13 @@ function getDisplayInfoBinary() {
     displayInfoBinaryPath = binPath;
     return binPath;
 }
-function isFiniteNumber(value) {
+function device_isFiniteNumber(value) {
     return 'number' == typeof value && Number.isFinite(value);
 }
-function isDisplayBounds(value) {
-    if (!value || 'object' != typeof value) return false;
-    const bounds = value;
-    return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
-}
-function isWindowsDisplayGeometry(value) {
-    if (!value || 'object' != typeof value) return false;
-    const candidate = value;
-    return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
-}
 function isDarwinDisplayGeometry(value) {
     if (!value || 'object' != typeof value) return false;
     const candidate = value;
-    return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
+    return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && device_isFiniteNumber(candidate.bounds.x) && device_isFiniteNumber(candidate.bounds.y) && device_isFiniteNumber(candidate.bounds.width) && device_isFiniteNumber(candidate.bounds.height);
 }
 function readDarwinDisplayGeometries() {
     const bin = getDisplayInfoBinary();
@@ -1007,11 +1109,6 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
     if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
     return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
 }
-function resolveWindowsDisplayGeometryFromList(displayId, displays) {
-    if (!displays.length) return;
-    if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
-    return displays.find((display)=>display.id === displayId);
-}
 function resolveDisplayGeometry(displayId, windowsDisplays) {
     if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
     if ('win32' === process.platform) {
@@ -1104,13 +1201,24 @@ function normalizePrimaryKey(key) {
     return KEY_NAME_MAP[lowerKey] || lowerKey;
 }
 class ComputerDevice {
+    usesPhysicalWindowsCoordinates() {
+        return 'win32' === process.platform && 'physical' === this.windowsCoordinateContext.mode;
+    }
+    assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
+        if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateContext.mode) return;
+        assertLegacyWindowsCoordinateCompatibility({
+            geometry: this.displayGeometry,
+            screenshotBase64,
+            inputSize: this.inputDriver.getScreenSize()
+        });
+    }
     async moveGlobalPointer(point, context, smooth) {
         if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
         const target = {
             x: Math.round(point.x),
             y: Math.round(point.y)
         };
-        if ('win32' === process.platform) {
+        if (this.usesPhysicalWindowsCoordinates()) {
             const actual = this.windowsPointerDriver.moveTo(target, {
                 smoothSteps: smooth?.smoothSteps,
                 smoothDelayMs: smooth?.smoothDelay
@@ -1162,7 +1270,7 @@ class ComputerDevice {
     }
     static async listDisplays() {
         try {
-            if ('win32' === process.platform) return listWindowsDisplays();
+            if ('win32' === process.platform) return listWindowsDisplays(discoverWindowsDisplays().geometries);
             const displays = await screenshot_desktop.listDisplays();
             return displays.map((d)=>({
                     id: String(d.id),
@@ -1200,7 +1308,14 @@ class ComputerDevice {
                 }
             }
             device_libnut = await getLibnut();
-            const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
+            const windowsDisplayDiscovery = 'win32' === process.platform ? discoverWindowsDisplays() : void 0;
+            const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
+            if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows display enumeration returned no data in the Per-Monitor V2 DPI context; using the legacy Windows coordinate path for this device connection.');
+            this.windowsCoordinateContext = windowsDisplayDiscovery?.coordinateMode === 'legacy' ? {
+                mode: 'legacy'
+            } : {
+                mode: 'physical'
+            };
             this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
             const size = await this.size();
             const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
@@ -1211,6 +1326,7 @@ Platform: ${process.platform}
 Display: ${this.displayId || 'Primary'}
 Screen Size: ${size.width}x${size.height}
 Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
+${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateContext.mode}` : ''}
 `;
             debugDevice('Computer device connected', this.description);
             await this.healthCheck(displays);
@@ -1234,7 +1350,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
     }
     async healthCheck(displays) {
         console.log('[HealthCheck] Starting health check...');
-        console.log("[HealthCheck] @midscene/computer v1.13.0");
+        console.log("[HealthCheck] @midscene/computer v1.13.1-beta-20260921032526.0");
         console.log('[HealthCheck] Taking screenshot...');
         const screenshotTimeout = 15000;
         let timeoutId;
@@ -1246,10 +1362,12 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
             timeoutPromise
         ]);
         console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
+        this.assertLegacyWindowsCoordinateCompatibility(base64);
         console.log('[HealthCheck] Verifying mouse control...');
-        const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
+        const usePhysicalWindowsCoordinates = this.usesPhysicalWindowsCoordinates();
+        const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
         console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
-        if ('win32' === process.platform) {
+        if (usePhysicalWindowsCoordinates) {
             if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
             const bounds = this.displayGeometry.bounds;
             const target = {
@@ -1315,7 +1433,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
         debugDevice('Taking screenshot', {
             displayId: this.displayId
         });
-        if ('win32' === process.platform) return this.screenshotViaPowershell();
+        if (this.usesPhysicalWindowsCoordinates()) return this.screenshotViaPowershell();
         const options = {
             format: 'png'
         };
@@ -1478,7 +1596,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
         this.inputDriver.sendKey(key, modifiers);
     }
     resolveUntargetedScrollPoint(screenSize) {
-        if ('win32' === process.platform) {
+        if (this.usesPhysicalWindowsCoordinates()) {
             const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
             if (activeWindowRect) {
                 const activeWindowCenter = {
@@ -1622,6 +1740,9 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
         device_define_property(this, "options", void 0);
         device_define_property(this, "displayId", void 0);
         device_define_property(this, "displayGeometry", void 0);
+        device_define_property(this, "windowsCoordinateContext", {
+            mode: 'physical'
+        });
         device_define_property(this, "description", void 0);
         device_define_property(this, "destroyed", false);
         device_define_property(this, "xvfbInstance", void 0);
@@ -2723,7 +2844,7 @@ const tools = new ComputerMidsceneTools({
 });
 runToolsCLI(tools, 'midscene-computer', {
     stripPrefix: 'computer_',
-    version: "1.13.0",
+    version: "1.13.1-beta-20260921032526.0",
     extraCommands: createReportCliCommands()
 }).catch((e)=>{
     process.exit(reportCLIError(e));