@midscene/computer 1.13.1-beta-20260921075813.0 → 1.13.1-beta-20260922030100.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +144 -57
- package/dist/es/index.mjs +144 -57
- package/dist/lib/cli.js +144 -57
- package/dist/lib/index.js +144 -57
- package/dist/types/index.d.ts +3 -0
- package/package.json +3 -3
package/dist/es/cli.mjs
CHANGED
|
@@ -461,17 +461,11 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
461
461
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
462
462
|
}
|
|
463
463
|
`.trim();
|
|
464
|
-
function
|
|
465
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
466
|
-
$ErrorActionPreference = 'Stop'
|
|
467
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
468
|
-
${script}`;
|
|
469
|
-
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
464
|
+
function runWindowsPowershellCommand(script) {
|
|
470
465
|
return execFileSync('powershell.exe', [
|
|
471
466
|
'-NoProfile',
|
|
472
|
-
'-
|
|
473
|
-
|
|
474
|
-
encoded
|
|
467
|
+
'-Command',
|
|
468
|
+
script
|
|
475
469
|
], {
|
|
476
470
|
encoding: 'utf8',
|
|
477
471
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -479,6 +473,118 @@ ${script}`;
|
|
|
479
473
|
windowsHide: true
|
|
480
474
|
});
|
|
481
475
|
}
|
|
476
|
+
function runWindowsPowershell(script) {
|
|
477
|
+
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
478
|
+
$ErrorActionPreference = 'Stop'
|
|
479
|
+
${script}`);
|
|
480
|
+
}
|
|
481
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
482
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
483
|
+
$ErrorActionPreference = 'Stop'
|
|
484
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
485
|
+
${script}`;
|
|
486
|
+
return runWindowsPowershellCommand(physicalPixelScript);
|
|
487
|
+
}
|
|
488
|
+
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
489
|
+
}
|
|
490
|
+
function isFiniteNumber(value) {
|
|
491
|
+
return 'number' == typeof value && Number.isFinite(value);
|
|
492
|
+
}
|
|
493
|
+
function isWindowsDisplayGeometry(value) {
|
|
494
|
+
if (!value || 'object' != typeof value) return false;
|
|
495
|
+
const candidate = value;
|
|
496
|
+
const bounds = candidate.bounds;
|
|
497
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
498
|
+
}
|
|
499
|
+
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
500
|
+
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
501
|
+
const script = `
|
|
502
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
503
|
+
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
504
|
+
$bounds = $_.Bounds
|
|
505
|
+
[PSCustomObject]@{
|
|
506
|
+
id = $_.DeviceName
|
|
507
|
+
name = $_.DeviceName
|
|
508
|
+
primary = $_.Primary
|
|
509
|
+
bounds = [PSCustomObject]@{
|
|
510
|
+
x = $bounds.X
|
|
511
|
+
y = $bounds.Y
|
|
512
|
+
width = $bounds.Width
|
|
513
|
+
height = $bounds.Height
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
})
|
|
517
|
+
${serializeScreens}
|
|
518
|
+
`.trim();
|
|
519
|
+
const output = runPowershell(script).trim();
|
|
520
|
+
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
521
|
+
const parsed = JSON.parse(output);
|
|
522
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
523
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
524
|
+
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
525
|
+
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
526
|
+
return displays;
|
|
527
|
+
}
|
|
528
|
+
function readWindowsDisplayGeometries() {
|
|
529
|
+
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
530
|
+
}
|
|
531
|
+
function discoverWindowsDisplays(runners = {
|
|
532
|
+
physical: runWindowsPhysicalPixelPowershell,
|
|
533
|
+
legacy: runWindowsPowershell
|
|
534
|
+
}) {
|
|
535
|
+
try {
|
|
536
|
+
return {
|
|
537
|
+
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
538
|
+
coordinateMode: 'physical'
|
|
539
|
+
};
|
|
540
|
+
} catch (error) {
|
|
541
|
+
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
542
|
+
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
543
|
+
return {
|
|
544
|
+
geometries,
|
|
545
|
+
coordinateMode: 'legacy'
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
function readPngDimensions(dataUri) {
|
|
550
|
+
const prefix = 'data:image/png;base64,';
|
|
551
|
+
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
552
|
+
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
553
|
+
const pngSignature = Buffer.from([
|
|
554
|
+
137,
|
|
555
|
+
80,
|
|
556
|
+
78,
|
|
557
|
+
71,
|
|
558
|
+
13,
|
|
559
|
+
10,
|
|
560
|
+
26,
|
|
561
|
+
10
|
|
562
|
+
]);
|
|
563
|
+
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
564
|
+
const width = buffer.readUInt32BE(16);
|
|
565
|
+
const height = buffer.readUInt32BE(20);
|
|
566
|
+
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
567
|
+
return {
|
|
568
|
+
width,
|
|
569
|
+
height
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
573
|
+
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
574
|
+
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
575
|
+
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
576
|
+
const displaySize = {
|
|
577
|
+
width: Math.round(geometry.bounds.width),
|
|
578
|
+
height: Math.round(geometry.bounds.height)
|
|
579
|
+
};
|
|
580
|
+
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
581
|
+
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
582
|
+
}
|
|
583
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
584
|
+
if (!displays.length) return;
|
|
585
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
586
|
+
return displays.find((display)=>display.id === displayId);
|
|
587
|
+
}
|
|
482
588
|
function windows_pointer_define_property(obj, key, value) {
|
|
483
589
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
484
590
|
value: value,
|
|
@@ -796,28 +902,6 @@ const EDGE_SCROLL_SPEC = {
|
|
|
796
902
|
function escapePowershellSingleQuoted(value) {
|
|
797
903
|
return value.replace(/'/g, "''");
|
|
798
904
|
}
|
|
799
|
-
function readWindowsDisplayGeometries() {
|
|
800
|
-
const script = `
|
|
801
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
802
|
-
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
803
|
-
$b = $_.Bounds
|
|
804
|
-
[PSCustomObject]@{
|
|
805
|
-
id = $_.DeviceName
|
|
806
|
-
name = $_.DeviceName
|
|
807
|
-
primary = $_.Primary
|
|
808
|
-
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
809
|
-
}
|
|
810
|
-
}
|
|
811
|
-
ConvertTo-Json @($s) -Compress
|
|
812
|
-
`.trim();
|
|
813
|
-
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
814
|
-
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
815
|
-
const parsed = JSON.parse(output);
|
|
816
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
817
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
818
|
-
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
819
|
-
return displays;
|
|
820
|
-
}
|
|
821
905
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
822
906
|
return geometries.map((display)=>({
|
|
823
907
|
id: display.id,
|
|
@@ -921,23 +1005,13 @@ function getDisplayInfoBinary() {
|
|
|
921
1005
|
displayInfoBinaryPath = binPath;
|
|
922
1006
|
return binPath;
|
|
923
1007
|
}
|
|
924
|
-
function
|
|
1008
|
+
function device_isFiniteNumber(value) {
|
|
925
1009
|
return 'number' == typeof value && Number.isFinite(value);
|
|
926
1010
|
}
|
|
927
|
-
function isDisplayBounds(value) {
|
|
928
|
-
if (!value || 'object' != typeof value) return false;
|
|
929
|
-
const bounds = value;
|
|
930
|
-
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
931
|
-
}
|
|
932
|
-
function isWindowsDisplayGeometry(value) {
|
|
933
|
-
if (!value || 'object' != typeof value) return false;
|
|
934
|
-
const candidate = value;
|
|
935
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
936
|
-
}
|
|
937
1011
|
function isDarwinDisplayGeometry(value) {
|
|
938
1012
|
if (!value || 'object' != typeof value) return false;
|
|
939
1013
|
const candidate = value;
|
|
940
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
1014
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && device_isFiniteNumber(candidate.bounds.x) && device_isFiniteNumber(candidate.bounds.y) && device_isFiniteNumber(candidate.bounds.width) && device_isFiniteNumber(candidate.bounds.height);
|
|
941
1015
|
}
|
|
942
1016
|
function readDarwinDisplayGeometries() {
|
|
943
1017
|
const bin = getDisplayInfoBinary();
|
|
@@ -1007,11 +1081,6 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1007
1081
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1008
1082
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1009
1083
|
}
|
|
1010
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1011
|
-
if (!displays.length) return;
|
|
1012
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1013
|
-
return displays.find((display)=>display.id === displayId);
|
|
1014
|
-
}
|
|
1015
1084
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1016
1085
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1017
1086
|
if ('win32' === process.platform) {
|
|
@@ -1104,13 +1173,24 @@ function normalizePrimaryKey(key) {
|
|
|
1104
1173
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1105
1174
|
}
|
|
1106
1175
|
class ComputerDevice {
|
|
1176
|
+
usesPhysicalWindowsCoordinates() {
|
|
1177
|
+
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1178
|
+
}
|
|
1179
|
+
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1180
|
+
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1181
|
+
assertLegacyWindowsCoordinateCompatibility({
|
|
1182
|
+
geometry: this.displayGeometry,
|
|
1183
|
+
screenshotBase64,
|
|
1184
|
+
inputSize: this.inputDriver.getScreenSize()
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1107
1187
|
async moveGlobalPointer(point, context, smooth) {
|
|
1108
1188
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1109
1189
|
const target = {
|
|
1110
1190
|
x: Math.round(point.x),
|
|
1111
1191
|
y: Math.round(point.y)
|
|
1112
1192
|
};
|
|
1113
|
-
if (
|
|
1193
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1114
1194
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1115
1195
|
smoothSteps: smooth?.smoothSteps,
|
|
1116
1196
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1162,7 +1242,7 @@ class ComputerDevice {
|
|
|
1162
1242
|
}
|
|
1163
1243
|
static async listDisplays() {
|
|
1164
1244
|
try {
|
|
1165
|
-
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1245
|
+
if ('win32' === process.platform) return listWindowsDisplays(discoverWindowsDisplays().geometries);
|
|
1166
1246
|
const displays = await screenshot_desktop.listDisplays();
|
|
1167
1247
|
return displays.map((d)=>({
|
|
1168
1248
|
id: String(d.id),
|
|
@@ -1200,7 +1280,10 @@ class ComputerDevice {
|
|
|
1200
1280
|
}
|
|
1201
1281
|
}
|
|
1202
1282
|
device_libnut = await getLibnut();
|
|
1203
|
-
const
|
|
1283
|
+
const windowsDisplayDiscovery = 'win32' === process.platform ? discoverWindowsDisplays() : void 0;
|
|
1284
|
+
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1285
|
+
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1286
|
+
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1204
1287
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1205
1288
|
const size = await this.size();
|
|
1206
1289
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1211,6 +1294,7 @@ Platform: ${process.platform}
|
|
|
1211
1294
|
Display: ${this.displayId || 'Primary'}
|
|
1212
1295
|
Screen Size: ${size.width}x${size.height}
|
|
1213
1296
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1297
|
+
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1214
1298
|
`;
|
|
1215
1299
|
debugDevice('Computer device connected', this.description);
|
|
1216
1300
|
await this.healthCheck(displays);
|
|
@@ -1234,7 +1318,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1234
1318
|
}
|
|
1235
1319
|
async healthCheck(displays) {
|
|
1236
1320
|
console.log('[HealthCheck] Starting health check...');
|
|
1237
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-
|
|
1321
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-20260922030100.0");
|
|
1238
1322
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1239
1323
|
const screenshotTimeout = 15000;
|
|
1240
1324
|
let timeoutId;
|
|
@@ -1246,10 +1330,12 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1246
1330
|
timeoutPromise
|
|
1247
1331
|
]);
|
|
1248
1332
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1333
|
+
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1249
1334
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1250
|
-
const
|
|
1335
|
+
const usePhysicalWindowsCoordinates = this.usesPhysicalWindowsCoordinates();
|
|
1336
|
+
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1251
1337
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1252
|
-
if (
|
|
1338
|
+
if (usePhysicalWindowsCoordinates) {
|
|
1253
1339
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1254
1340
|
const bounds = this.displayGeometry.bounds;
|
|
1255
1341
|
const target = {
|
|
@@ -1315,7 +1401,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1315
1401
|
debugDevice('Taking screenshot', {
|
|
1316
1402
|
displayId: this.displayId
|
|
1317
1403
|
});
|
|
1318
|
-
if (
|
|
1404
|
+
if (this.usesPhysicalWindowsCoordinates()) return this.screenshotViaPowershell();
|
|
1319
1405
|
const options = {
|
|
1320
1406
|
format: 'png'
|
|
1321
1407
|
};
|
|
@@ -1478,7 +1564,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1478
1564
|
this.inputDriver.sendKey(key, modifiers);
|
|
1479
1565
|
}
|
|
1480
1566
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1481
|
-
if (
|
|
1567
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1482
1568
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1483
1569
|
if (activeWindowRect) {
|
|
1484
1570
|
const activeWindowCenter = {
|
|
@@ -1622,6 +1708,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1622
1708
|
device_define_property(this, "options", void 0);
|
|
1623
1709
|
device_define_property(this, "displayId", void 0);
|
|
1624
1710
|
device_define_property(this, "displayGeometry", void 0);
|
|
1711
|
+
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1625
1712
|
device_define_property(this, "description", void 0);
|
|
1626
1713
|
device_define_property(this, "destroyed", false);
|
|
1627
1714
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2723,7 +2810,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2723
2810
|
});
|
|
2724
2811
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2725
2812
|
stripPrefix: 'computer_',
|
|
2726
|
-
version: "1.13.1-beta-
|
|
2813
|
+
version: "1.13.1-beta-20260922030100.0",
|
|
2727
2814
|
extraCommands: createReportCliCommands()
|
|
2728
2815
|
}).catch((e)=>{
|
|
2729
2816
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -461,17 +461,11 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
461
461
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
462
462
|
}
|
|
463
463
|
`.trim();
|
|
464
|
-
function
|
|
465
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
466
|
-
$ErrorActionPreference = 'Stop'
|
|
467
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
468
|
-
${script}`;
|
|
469
|
-
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
464
|
+
function runWindowsPowershellCommand(script) {
|
|
470
465
|
return execFileSync('powershell.exe', [
|
|
471
466
|
'-NoProfile',
|
|
472
|
-
'-
|
|
473
|
-
|
|
474
|
-
encoded
|
|
467
|
+
'-Command',
|
|
468
|
+
script
|
|
475
469
|
], {
|
|
476
470
|
encoding: 'utf8',
|
|
477
471
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -479,6 +473,118 @@ ${script}`;
|
|
|
479
473
|
windowsHide: true
|
|
480
474
|
});
|
|
481
475
|
}
|
|
476
|
+
function runWindowsPowershell(script) {
|
|
477
|
+
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
478
|
+
$ErrorActionPreference = 'Stop'
|
|
479
|
+
${script}`);
|
|
480
|
+
}
|
|
481
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
482
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
483
|
+
$ErrorActionPreference = 'Stop'
|
|
484
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
485
|
+
${script}`;
|
|
486
|
+
return runWindowsPowershellCommand(physicalPixelScript);
|
|
487
|
+
}
|
|
488
|
+
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
489
|
+
}
|
|
490
|
+
function isFiniteNumber(value) {
|
|
491
|
+
return 'number' == typeof value && Number.isFinite(value);
|
|
492
|
+
}
|
|
493
|
+
function isWindowsDisplayGeometry(value) {
|
|
494
|
+
if (!value || 'object' != typeof value) return false;
|
|
495
|
+
const candidate = value;
|
|
496
|
+
const bounds = candidate.bounds;
|
|
497
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
498
|
+
}
|
|
499
|
+
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
500
|
+
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
501
|
+
const script = `
|
|
502
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
503
|
+
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
504
|
+
$bounds = $_.Bounds
|
|
505
|
+
[PSCustomObject]@{
|
|
506
|
+
id = $_.DeviceName
|
|
507
|
+
name = $_.DeviceName
|
|
508
|
+
primary = $_.Primary
|
|
509
|
+
bounds = [PSCustomObject]@{
|
|
510
|
+
x = $bounds.X
|
|
511
|
+
y = $bounds.Y
|
|
512
|
+
width = $bounds.Width
|
|
513
|
+
height = $bounds.Height
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
})
|
|
517
|
+
${serializeScreens}
|
|
518
|
+
`.trim();
|
|
519
|
+
const output = runPowershell(script).trim();
|
|
520
|
+
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
521
|
+
const parsed = JSON.parse(output);
|
|
522
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
523
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
524
|
+
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
525
|
+
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
526
|
+
return displays;
|
|
527
|
+
}
|
|
528
|
+
function readWindowsDisplayGeometries() {
|
|
529
|
+
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
530
|
+
}
|
|
531
|
+
function discoverWindowsDisplays(runners = {
|
|
532
|
+
physical: runWindowsPhysicalPixelPowershell,
|
|
533
|
+
legacy: runWindowsPowershell
|
|
534
|
+
}) {
|
|
535
|
+
try {
|
|
536
|
+
return {
|
|
537
|
+
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
538
|
+
coordinateMode: 'physical'
|
|
539
|
+
};
|
|
540
|
+
} catch (error) {
|
|
541
|
+
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
542
|
+
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
543
|
+
return {
|
|
544
|
+
geometries,
|
|
545
|
+
coordinateMode: 'legacy'
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
function readPngDimensions(dataUri) {
|
|
550
|
+
const prefix = 'data:image/png;base64,';
|
|
551
|
+
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
552
|
+
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
553
|
+
const pngSignature = Buffer.from([
|
|
554
|
+
137,
|
|
555
|
+
80,
|
|
556
|
+
78,
|
|
557
|
+
71,
|
|
558
|
+
13,
|
|
559
|
+
10,
|
|
560
|
+
26,
|
|
561
|
+
10
|
|
562
|
+
]);
|
|
563
|
+
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
564
|
+
const width = buffer.readUInt32BE(16);
|
|
565
|
+
const height = buffer.readUInt32BE(20);
|
|
566
|
+
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
567
|
+
return {
|
|
568
|
+
width,
|
|
569
|
+
height
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
573
|
+
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
574
|
+
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
575
|
+
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
576
|
+
const displaySize = {
|
|
577
|
+
width: Math.round(geometry.bounds.width),
|
|
578
|
+
height: Math.round(geometry.bounds.height)
|
|
579
|
+
};
|
|
580
|
+
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
581
|
+
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
582
|
+
}
|
|
583
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
584
|
+
if (!displays.length) return;
|
|
585
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
586
|
+
return displays.find((display)=>display.id === displayId);
|
|
587
|
+
}
|
|
482
588
|
function windows_pointer_define_property(obj, key, value) {
|
|
483
589
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
484
590
|
value: value,
|
|
@@ -796,28 +902,6 @@ const EDGE_SCROLL_SPEC = {
|
|
|
796
902
|
function escapePowershellSingleQuoted(value) {
|
|
797
903
|
return value.replace(/'/g, "''");
|
|
798
904
|
}
|
|
799
|
-
function readWindowsDisplayGeometries() {
|
|
800
|
-
const script = `
|
|
801
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
802
|
-
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
803
|
-
$b = $_.Bounds
|
|
804
|
-
[PSCustomObject]@{
|
|
805
|
-
id = $_.DeviceName
|
|
806
|
-
name = $_.DeviceName
|
|
807
|
-
primary = $_.Primary
|
|
808
|
-
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
809
|
-
}
|
|
810
|
-
}
|
|
811
|
-
ConvertTo-Json @($s) -Compress
|
|
812
|
-
`.trim();
|
|
813
|
-
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
814
|
-
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
815
|
-
const parsed = JSON.parse(output);
|
|
816
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
817
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
818
|
-
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
819
|
-
return displays;
|
|
820
|
-
}
|
|
821
905
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
822
906
|
return geometries.map((display)=>({
|
|
823
907
|
id: display.id,
|
|
@@ -921,23 +1005,13 @@ function getDisplayInfoBinary() {
|
|
|
921
1005
|
displayInfoBinaryPath = binPath;
|
|
922
1006
|
return binPath;
|
|
923
1007
|
}
|
|
924
|
-
function
|
|
1008
|
+
function device_isFiniteNumber(value) {
|
|
925
1009
|
return 'number' == typeof value && Number.isFinite(value);
|
|
926
1010
|
}
|
|
927
|
-
function isDisplayBounds(value) {
|
|
928
|
-
if (!value || 'object' != typeof value) return false;
|
|
929
|
-
const bounds = value;
|
|
930
|
-
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
931
|
-
}
|
|
932
|
-
function isWindowsDisplayGeometry(value) {
|
|
933
|
-
if (!value || 'object' != typeof value) return false;
|
|
934
|
-
const candidate = value;
|
|
935
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
936
|
-
}
|
|
937
1011
|
function isDarwinDisplayGeometry(value) {
|
|
938
1012
|
if (!value || 'object' != typeof value) return false;
|
|
939
1013
|
const candidate = value;
|
|
940
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
1014
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && device_isFiniteNumber(candidate.bounds.x) && device_isFiniteNumber(candidate.bounds.y) && device_isFiniteNumber(candidate.bounds.width) && device_isFiniteNumber(candidate.bounds.height);
|
|
941
1015
|
}
|
|
942
1016
|
function readDarwinDisplayGeometries() {
|
|
943
1017
|
const bin = getDisplayInfoBinary();
|
|
@@ -1007,11 +1081,6 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1007
1081
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1008
1082
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1009
1083
|
}
|
|
1010
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1011
|
-
if (!displays.length) return;
|
|
1012
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1013
|
-
return displays.find((display)=>display.id === displayId);
|
|
1014
|
-
}
|
|
1015
1084
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1016
1085
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1017
1086
|
if ('win32' === process.platform) {
|
|
@@ -1104,13 +1173,24 @@ function normalizePrimaryKey(key) {
|
|
|
1104
1173
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1105
1174
|
}
|
|
1106
1175
|
class ComputerDevice {
|
|
1176
|
+
usesPhysicalWindowsCoordinates() {
|
|
1177
|
+
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1178
|
+
}
|
|
1179
|
+
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1180
|
+
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1181
|
+
assertLegacyWindowsCoordinateCompatibility({
|
|
1182
|
+
geometry: this.displayGeometry,
|
|
1183
|
+
screenshotBase64,
|
|
1184
|
+
inputSize: this.inputDriver.getScreenSize()
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1107
1187
|
async moveGlobalPointer(point, context, smooth) {
|
|
1108
1188
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1109
1189
|
const target = {
|
|
1110
1190
|
x: Math.round(point.x),
|
|
1111
1191
|
y: Math.round(point.y)
|
|
1112
1192
|
};
|
|
1113
|
-
if (
|
|
1193
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1114
1194
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1115
1195
|
smoothSteps: smooth?.smoothSteps,
|
|
1116
1196
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1162,7 +1242,7 @@ class ComputerDevice {
|
|
|
1162
1242
|
}
|
|
1163
1243
|
static async listDisplays() {
|
|
1164
1244
|
try {
|
|
1165
|
-
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1245
|
+
if ('win32' === process.platform) return listWindowsDisplays(discoverWindowsDisplays().geometries);
|
|
1166
1246
|
const displays = await screenshot_desktop.listDisplays();
|
|
1167
1247
|
return displays.map((d)=>({
|
|
1168
1248
|
id: String(d.id),
|
|
@@ -1200,7 +1280,10 @@ class ComputerDevice {
|
|
|
1200
1280
|
}
|
|
1201
1281
|
}
|
|
1202
1282
|
device_libnut = await getLibnut();
|
|
1203
|
-
const
|
|
1283
|
+
const windowsDisplayDiscovery = 'win32' === process.platform ? discoverWindowsDisplays() : void 0;
|
|
1284
|
+
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1285
|
+
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1286
|
+
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1204
1287
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1205
1288
|
const size = await this.size();
|
|
1206
1289
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1211,6 +1294,7 @@ Platform: ${process.platform}
|
|
|
1211
1294
|
Display: ${this.displayId || 'Primary'}
|
|
1212
1295
|
Screen Size: ${size.width}x${size.height}
|
|
1213
1296
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1297
|
+
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1214
1298
|
`;
|
|
1215
1299
|
debugDevice('Computer device connected', this.description);
|
|
1216
1300
|
await this.healthCheck(displays);
|
|
@@ -1234,7 +1318,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1234
1318
|
}
|
|
1235
1319
|
async healthCheck(displays) {
|
|
1236
1320
|
console.log('[HealthCheck] Starting health check...');
|
|
1237
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-
|
|
1321
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-20260922030100.0");
|
|
1238
1322
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1239
1323
|
const screenshotTimeout = 15000;
|
|
1240
1324
|
let timeoutId;
|
|
@@ -1246,10 +1330,12 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1246
1330
|
timeoutPromise
|
|
1247
1331
|
]);
|
|
1248
1332
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1333
|
+
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1249
1334
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1250
|
-
const
|
|
1335
|
+
const usePhysicalWindowsCoordinates = this.usesPhysicalWindowsCoordinates();
|
|
1336
|
+
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1251
1337
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1252
|
-
if (
|
|
1338
|
+
if (usePhysicalWindowsCoordinates) {
|
|
1253
1339
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1254
1340
|
const bounds = this.displayGeometry.bounds;
|
|
1255
1341
|
const target = {
|
|
@@ -1315,7 +1401,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1315
1401
|
debugDevice('Taking screenshot', {
|
|
1316
1402
|
displayId: this.displayId
|
|
1317
1403
|
});
|
|
1318
|
-
if (
|
|
1404
|
+
if (this.usesPhysicalWindowsCoordinates()) return this.screenshotViaPowershell();
|
|
1319
1405
|
const options = {
|
|
1320
1406
|
format: 'png'
|
|
1321
1407
|
};
|
|
@@ -1478,7 +1564,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1478
1564
|
this.inputDriver.sendKey(key, modifiers);
|
|
1479
1565
|
}
|
|
1480
1566
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1481
|
-
if (
|
|
1567
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1482
1568
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1483
1569
|
if (activeWindowRect) {
|
|
1484
1570
|
const activeWindowCenter = {
|
|
@@ -1622,6 +1708,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1622
1708
|
device_define_property(this, "options", void 0);
|
|
1623
1709
|
device_define_property(this, "displayId", void 0);
|
|
1624
1710
|
device_define_property(this, "displayGeometry", void 0);
|
|
1711
|
+
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1625
1712
|
device_define_property(this, "description", void 0);
|
|
1626
1713
|
device_define_property(this, "destroyed", false);
|
|
1627
1714
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2762,7 +2849,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2762
2849
|
}
|
|
2763
2850
|
}
|
|
2764
2851
|
function version() {
|
|
2765
|
-
const currentVersion = "1.13.1-beta-
|
|
2852
|
+
const currentVersion = "1.13.1-beta-20260922030100.0";
|
|
2766
2853
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2767
2854
|
return currentVersion;
|
|
2768
2855
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -486,17 +486,11 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
486
486
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
487
487
|
}
|
|
488
488
|
`.trim();
|
|
489
|
-
function
|
|
490
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
491
|
-
$ErrorActionPreference = 'Stop'
|
|
492
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
493
|
-
${script}`;
|
|
494
|
-
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
489
|
+
function runWindowsPowershellCommand(script) {
|
|
495
490
|
return (0, external_node_child_process_namespaceObject.execFileSync)('powershell.exe', [
|
|
496
491
|
'-NoProfile',
|
|
497
|
-
'-
|
|
498
|
-
|
|
499
|
-
encoded
|
|
492
|
+
'-Command',
|
|
493
|
+
script
|
|
500
494
|
], {
|
|
501
495
|
encoding: 'utf8',
|
|
502
496
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -504,6 +498,118 @@ ${script}`;
|
|
|
504
498
|
windowsHide: true
|
|
505
499
|
});
|
|
506
500
|
}
|
|
501
|
+
function runWindowsPowershell(script) {
|
|
502
|
+
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
503
|
+
$ErrorActionPreference = 'Stop'
|
|
504
|
+
${script}`);
|
|
505
|
+
}
|
|
506
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
507
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
508
|
+
$ErrorActionPreference = 'Stop'
|
|
509
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
510
|
+
${script}`;
|
|
511
|
+
return runWindowsPowershellCommand(physicalPixelScript);
|
|
512
|
+
}
|
|
513
|
+
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
514
|
+
}
|
|
515
|
+
function isFiniteNumber(value) {
|
|
516
|
+
return 'number' == typeof value && Number.isFinite(value);
|
|
517
|
+
}
|
|
518
|
+
function isWindowsDisplayGeometry(value) {
|
|
519
|
+
if (!value || 'object' != typeof value) return false;
|
|
520
|
+
const candidate = value;
|
|
521
|
+
const bounds = candidate.bounds;
|
|
522
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
523
|
+
}
|
|
524
|
+
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
525
|
+
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
526
|
+
const script = `
|
|
527
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
528
|
+
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
529
|
+
$bounds = $_.Bounds
|
|
530
|
+
[PSCustomObject]@{
|
|
531
|
+
id = $_.DeviceName
|
|
532
|
+
name = $_.DeviceName
|
|
533
|
+
primary = $_.Primary
|
|
534
|
+
bounds = [PSCustomObject]@{
|
|
535
|
+
x = $bounds.X
|
|
536
|
+
y = $bounds.Y
|
|
537
|
+
width = $bounds.Width
|
|
538
|
+
height = $bounds.Height
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
})
|
|
542
|
+
${serializeScreens}
|
|
543
|
+
`.trim();
|
|
544
|
+
const output = runPowershell(script).trim();
|
|
545
|
+
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
546
|
+
const parsed = JSON.parse(output);
|
|
547
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
548
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
549
|
+
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
550
|
+
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
551
|
+
return displays;
|
|
552
|
+
}
|
|
553
|
+
function readWindowsDisplayGeometries() {
|
|
554
|
+
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
555
|
+
}
|
|
556
|
+
function discoverWindowsDisplays(runners = {
|
|
557
|
+
physical: runWindowsPhysicalPixelPowershell,
|
|
558
|
+
legacy: runWindowsPowershell
|
|
559
|
+
}) {
|
|
560
|
+
try {
|
|
561
|
+
return {
|
|
562
|
+
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
563
|
+
coordinateMode: 'physical'
|
|
564
|
+
};
|
|
565
|
+
} catch (error) {
|
|
566
|
+
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
567
|
+
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
568
|
+
return {
|
|
569
|
+
geometries,
|
|
570
|
+
coordinateMode: 'legacy'
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
function readPngDimensions(dataUri) {
|
|
575
|
+
const prefix = 'data:image/png;base64,';
|
|
576
|
+
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
577
|
+
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
578
|
+
const pngSignature = Buffer.from([
|
|
579
|
+
137,
|
|
580
|
+
80,
|
|
581
|
+
78,
|
|
582
|
+
71,
|
|
583
|
+
13,
|
|
584
|
+
10,
|
|
585
|
+
26,
|
|
586
|
+
10
|
|
587
|
+
]);
|
|
588
|
+
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
589
|
+
const width = buffer.readUInt32BE(16);
|
|
590
|
+
const height = buffer.readUInt32BE(20);
|
|
591
|
+
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
592
|
+
return {
|
|
593
|
+
width,
|
|
594
|
+
height
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
598
|
+
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
599
|
+
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
600
|
+
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
601
|
+
const displaySize = {
|
|
602
|
+
width: Math.round(geometry.bounds.width),
|
|
603
|
+
height: Math.round(geometry.bounds.height)
|
|
604
|
+
};
|
|
605
|
+
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
606
|
+
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
607
|
+
}
|
|
608
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
609
|
+
if (!displays.length) return;
|
|
610
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
611
|
+
return displays.find((display)=>display.id === displayId);
|
|
612
|
+
}
|
|
507
613
|
function windows_pointer_define_property(obj, key, value) {
|
|
508
614
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
509
615
|
value: value,
|
|
@@ -822,28 +928,6 @@ const EDGE_SCROLL_SPEC = {
|
|
|
822
928
|
function escapePowershellSingleQuoted(value) {
|
|
823
929
|
return value.replace(/'/g, "''");
|
|
824
930
|
}
|
|
825
|
-
function readWindowsDisplayGeometries() {
|
|
826
|
-
const script = `
|
|
827
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
828
|
-
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
829
|
-
$b = $_.Bounds
|
|
830
|
-
[PSCustomObject]@{
|
|
831
|
-
id = $_.DeviceName
|
|
832
|
-
name = $_.DeviceName
|
|
833
|
-
primary = $_.Primary
|
|
834
|
-
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
ConvertTo-Json @($s) -Compress
|
|
838
|
-
`.trim();
|
|
839
|
-
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
840
|
-
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
841
|
-
const parsed = JSON.parse(output);
|
|
842
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
843
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
844
|
-
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
845
|
-
return displays;
|
|
846
|
-
}
|
|
847
931
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
848
932
|
return geometries.map((display)=>({
|
|
849
933
|
id: display.id,
|
|
@@ -947,23 +1031,13 @@ function getDisplayInfoBinary() {
|
|
|
947
1031
|
displayInfoBinaryPath = binPath;
|
|
948
1032
|
return binPath;
|
|
949
1033
|
}
|
|
950
|
-
function
|
|
1034
|
+
function device_isFiniteNumber(value) {
|
|
951
1035
|
return 'number' == typeof value && Number.isFinite(value);
|
|
952
1036
|
}
|
|
953
|
-
function isDisplayBounds(value) {
|
|
954
|
-
if (!value || 'object' != typeof value) return false;
|
|
955
|
-
const bounds = value;
|
|
956
|
-
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
957
|
-
}
|
|
958
|
-
function isWindowsDisplayGeometry(value) {
|
|
959
|
-
if (!value || 'object' != typeof value) return false;
|
|
960
|
-
const candidate = value;
|
|
961
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
962
|
-
}
|
|
963
1037
|
function isDarwinDisplayGeometry(value) {
|
|
964
1038
|
if (!value || 'object' != typeof value) return false;
|
|
965
1039
|
const candidate = value;
|
|
966
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
1040
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && device_isFiniteNumber(candidate.bounds.x) && device_isFiniteNumber(candidate.bounds.y) && device_isFiniteNumber(candidate.bounds.width) && device_isFiniteNumber(candidate.bounds.height);
|
|
967
1041
|
}
|
|
968
1042
|
function readDarwinDisplayGeometries() {
|
|
969
1043
|
const bin = getDisplayInfoBinary();
|
|
@@ -1033,11 +1107,6 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1033
1107
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1034
1108
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1035
1109
|
}
|
|
1036
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1037
|
-
if (!displays.length) return;
|
|
1038
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1039
|
-
return displays.find((display)=>display.id === displayId);
|
|
1040
|
-
}
|
|
1041
1110
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1042
1111
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1043
1112
|
if ('win32' === process.platform) {
|
|
@@ -1130,13 +1199,24 @@ function normalizePrimaryKey(key) {
|
|
|
1130
1199
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1131
1200
|
}
|
|
1132
1201
|
class ComputerDevice {
|
|
1202
|
+
usesPhysicalWindowsCoordinates() {
|
|
1203
|
+
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1204
|
+
}
|
|
1205
|
+
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1206
|
+
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1207
|
+
assertLegacyWindowsCoordinateCompatibility({
|
|
1208
|
+
geometry: this.displayGeometry,
|
|
1209
|
+
screenshotBase64,
|
|
1210
|
+
inputSize: this.inputDriver.getScreenSize()
|
|
1211
|
+
});
|
|
1212
|
+
}
|
|
1133
1213
|
async moveGlobalPointer(point, context, smooth) {
|
|
1134
1214
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1135
1215
|
const target = {
|
|
1136
1216
|
x: Math.round(point.x),
|
|
1137
1217
|
y: Math.round(point.y)
|
|
1138
1218
|
};
|
|
1139
|
-
if (
|
|
1219
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1140
1220
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1141
1221
|
smoothSteps: smooth?.smoothSteps,
|
|
1142
1222
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1188,7 +1268,7 @@ class ComputerDevice {
|
|
|
1188
1268
|
}
|
|
1189
1269
|
static async listDisplays() {
|
|
1190
1270
|
try {
|
|
1191
|
-
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1271
|
+
if ('win32' === process.platform) return listWindowsDisplays(discoverWindowsDisplays().geometries);
|
|
1192
1272
|
const displays = await external_screenshot_desktop_default().listDisplays();
|
|
1193
1273
|
return displays.map((d)=>({
|
|
1194
1274
|
id: String(d.id),
|
|
@@ -1226,7 +1306,10 @@ class ComputerDevice {
|
|
|
1226
1306
|
}
|
|
1227
1307
|
}
|
|
1228
1308
|
device_libnut = await getLibnut();
|
|
1229
|
-
const
|
|
1309
|
+
const windowsDisplayDiscovery = 'win32' === process.platform ? discoverWindowsDisplays() : void 0;
|
|
1310
|
+
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1311
|
+
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1312
|
+
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1230
1313
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1231
1314
|
const size = await this.size();
|
|
1232
1315
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1237,6 +1320,7 @@ Platform: ${process.platform}
|
|
|
1237
1320
|
Display: ${this.displayId || 'Primary'}
|
|
1238
1321
|
Screen Size: ${size.width}x${size.height}
|
|
1239
1322
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1323
|
+
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1240
1324
|
`;
|
|
1241
1325
|
debugDevice('Computer device connected', this.description);
|
|
1242
1326
|
await this.healthCheck(displays);
|
|
@@ -1260,7 +1344,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1260
1344
|
}
|
|
1261
1345
|
async healthCheck(displays) {
|
|
1262
1346
|
console.log('[HealthCheck] Starting health check...');
|
|
1263
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-
|
|
1347
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-20260922030100.0");
|
|
1264
1348
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1265
1349
|
const screenshotTimeout = 15000;
|
|
1266
1350
|
let timeoutId;
|
|
@@ -1272,10 +1356,12 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1272
1356
|
timeoutPromise
|
|
1273
1357
|
]);
|
|
1274
1358
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1359
|
+
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1275
1360
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1276
|
-
const
|
|
1361
|
+
const usePhysicalWindowsCoordinates = this.usesPhysicalWindowsCoordinates();
|
|
1362
|
+
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1277
1363
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1278
|
-
if (
|
|
1364
|
+
if (usePhysicalWindowsCoordinates) {
|
|
1279
1365
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1280
1366
|
const bounds = this.displayGeometry.bounds;
|
|
1281
1367
|
const target = {
|
|
@@ -1341,7 +1427,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1341
1427
|
debugDevice('Taking screenshot', {
|
|
1342
1428
|
displayId: this.displayId
|
|
1343
1429
|
});
|
|
1344
|
-
if (
|
|
1430
|
+
if (this.usesPhysicalWindowsCoordinates()) return this.screenshotViaPowershell();
|
|
1345
1431
|
const options = {
|
|
1346
1432
|
format: 'png'
|
|
1347
1433
|
};
|
|
@@ -1504,7 +1590,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1504
1590
|
this.inputDriver.sendKey(key, modifiers);
|
|
1505
1591
|
}
|
|
1506
1592
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1507
|
-
if (
|
|
1593
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1508
1594
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1509
1595
|
if (activeWindowRect) {
|
|
1510
1596
|
const activeWindowCenter = {
|
|
@@ -1648,6 +1734,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1648
1734
|
device_define_property(this, "options", void 0);
|
|
1649
1735
|
device_define_property(this, "displayId", void 0);
|
|
1650
1736
|
device_define_property(this, "displayGeometry", void 0);
|
|
1737
|
+
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1651
1738
|
device_define_property(this, "description", void 0);
|
|
1652
1739
|
device_define_property(this, "destroyed", false);
|
|
1653
1740
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2750,7 +2837,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2750
2837
|
});
|
|
2751
2838
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
2752
2839
|
stripPrefix: 'computer_',
|
|
2753
|
-
version: "1.13.1-beta-
|
|
2840
|
+
version: "1.13.1-beta-20260922030100.0",
|
|
2754
2841
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
2755
2842
|
}).catch((e)=>{
|
|
2756
2843
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|
package/dist/lib/index.js
CHANGED
|
@@ -512,17 +512,11 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
512
512
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
513
513
|
}
|
|
514
514
|
`.trim();
|
|
515
|
-
function
|
|
516
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
517
|
-
$ErrorActionPreference = 'Stop'
|
|
518
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
519
|
-
${script}`;
|
|
520
|
-
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
515
|
+
function runWindowsPowershellCommand(script) {
|
|
521
516
|
return (0, external_node_child_process_namespaceObject.execFileSync)('powershell.exe', [
|
|
522
517
|
'-NoProfile',
|
|
523
|
-
'-
|
|
524
|
-
|
|
525
|
-
encoded
|
|
518
|
+
'-Command',
|
|
519
|
+
script
|
|
526
520
|
], {
|
|
527
521
|
encoding: 'utf8',
|
|
528
522
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -530,6 +524,118 @@ ${script}`;
|
|
|
530
524
|
windowsHide: true
|
|
531
525
|
});
|
|
532
526
|
}
|
|
527
|
+
function runWindowsPowershell(script) {
|
|
528
|
+
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
529
|
+
$ErrorActionPreference = 'Stop'
|
|
530
|
+
${script}`);
|
|
531
|
+
}
|
|
532
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
533
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
534
|
+
$ErrorActionPreference = 'Stop'
|
|
535
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
536
|
+
${script}`;
|
|
537
|
+
return runWindowsPowershellCommand(physicalPixelScript);
|
|
538
|
+
}
|
|
539
|
+
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
540
|
+
}
|
|
541
|
+
function isFiniteNumber(value) {
|
|
542
|
+
return 'number' == typeof value && Number.isFinite(value);
|
|
543
|
+
}
|
|
544
|
+
function isWindowsDisplayGeometry(value) {
|
|
545
|
+
if (!value || 'object' != typeof value) return false;
|
|
546
|
+
const candidate = value;
|
|
547
|
+
const bounds = candidate.bounds;
|
|
548
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
549
|
+
}
|
|
550
|
+
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
551
|
+
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
552
|
+
const script = `
|
|
553
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
554
|
+
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
555
|
+
$bounds = $_.Bounds
|
|
556
|
+
[PSCustomObject]@{
|
|
557
|
+
id = $_.DeviceName
|
|
558
|
+
name = $_.DeviceName
|
|
559
|
+
primary = $_.Primary
|
|
560
|
+
bounds = [PSCustomObject]@{
|
|
561
|
+
x = $bounds.X
|
|
562
|
+
y = $bounds.Y
|
|
563
|
+
width = $bounds.Width
|
|
564
|
+
height = $bounds.Height
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
})
|
|
568
|
+
${serializeScreens}
|
|
569
|
+
`.trim();
|
|
570
|
+
const output = runPowershell(script).trim();
|
|
571
|
+
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
572
|
+
const parsed = JSON.parse(output);
|
|
573
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
574
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
575
|
+
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
576
|
+
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
577
|
+
return displays;
|
|
578
|
+
}
|
|
579
|
+
function readWindowsDisplayGeometries() {
|
|
580
|
+
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
581
|
+
}
|
|
582
|
+
function discoverWindowsDisplays(runners = {
|
|
583
|
+
physical: runWindowsPhysicalPixelPowershell,
|
|
584
|
+
legacy: runWindowsPowershell
|
|
585
|
+
}) {
|
|
586
|
+
try {
|
|
587
|
+
return {
|
|
588
|
+
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
589
|
+
coordinateMode: 'physical'
|
|
590
|
+
};
|
|
591
|
+
} catch (error) {
|
|
592
|
+
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
593
|
+
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
594
|
+
return {
|
|
595
|
+
geometries,
|
|
596
|
+
coordinateMode: 'legacy'
|
|
597
|
+
};
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
function readPngDimensions(dataUri) {
|
|
601
|
+
const prefix = 'data:image/png;base64,';
|
|
602
|
+
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
603
|
+
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
604
|
+
const pngSignature = Buffer.from([
|
|
605
|
+
137,
|
|
606
|
+
80,
|
|
607
|
+
78,
|
|
608
|
+
71,
|
|
609
|
+
13,
|
|
610
|
+
10,
|
|
611
|
+
26,
|
|
612
|
+
10
|
|
613
|
+
]);
|
|
614
|
+
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
615
|
+
const width = buffer.readUInt32BE(16);
|
|
616
|
+
const height = buffer.readUInt32BE(20);
|
|
617
|
+
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
618
|
+
return {
|
|
619
|
+
width,
|
|
620
|
+
height
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
624
|
+
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
625
|
+
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
626
|
+
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
627
|
+
const displaySize = {
|
|
628
|
+
width: Math.round(geometry.bounds.width),
|
|
629
|
+
height: Math.round(geometry.bounds.height)
|
|
630
|
+
};
|
|
631
|
+
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
632
|
+
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
633
|
+
}
|
|
634
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
635
|
+
if (!displays.length) return;
|
|
636
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
637
|
+
return displays.find((display)=>display.id === displayId);
|
|
638
|
+
}
|
|
533
639
|
function windows_pointer_define_property(obj, key, value) {
|
|
534
640
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
535
641
|
value: value,
|
|
@@ -848,28 +954,6 @@ const EDGE_SCROLL_SPEC = {
|
|
|
848
954
|
function escapePowershellSingleQuoted(value) {
|
|
849
955
|
return value.replace(/'/g, "''");
|
|
850
956
|
}
|
|
851
|
-
function readWindowsDisplayGeometries() {
|
|
852
|
-
const script = `
|
|
853
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
854
|
-
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
855
|
-
$b = $_.Bounds
|
|
856
|
-
[PSCustomObject]@{
|
|
857
|
-
id = $_.DeviceName
|
|
858
|
-
name = $_.DeviceName
|
|
859
|
-
primary = $_.Primary
|
|
860
|
-
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
861
|
-
}
|
|
862
|
-
}
|
|
863
|
-
ConvertTo-Json @($s) -Compress
|
|
864
|
-
`.trim();
|
|
865
|
-
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
866
|
-
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
867
|
-
const parsed = JSON.parse(output);
|
|
868
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
869
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
870
|
-
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
871
|
-
return displays;
|
|
872
|
-
}
|
|
873
957
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
874
958
|
return geometries.map((display)=>({
|
|
875
959
|
id: display.id,
|
|
@@ -973,23 +1057,13 @@ function getDisplayInfoBinary() {
|
|
|
973
1057
|
displayInfoBinaryPath = binPath;
|
|
974
1058
|
return binPath;
|
|
975
1059
|
}
|
|
976
|
-
function
|
|
1060
|
+
function device_isFiniteNumber(value) {
|
|
977
1061
|
return 'number' == typeof value && Number.isFinite(value);
|
|
978
1062
|
}
|
|
979
|
-
function isDisplayBounds(value) {
|
|
980
|
-
if (!value || 'object' != typeof value) return false;
|
|
981
|
-
const bounds = value;
|
|
982
|
-
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
983
|
-
}
|
|
984
|
-
function isWindowsDisplayGeometry(value) {
|
|
985
|
-
if (!value || 'object' != typeof value) return false;
|
|
986
|
-
const candidate = value;
|
|
987
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
988
|
-
}
|
|
989
1063
|
function isDarwinDisplayGeometry(value) {
|
|
990
1064
|
if (!value || 'object' != typeof value) return false;
|
|
991
1065
|
const candidate = value;
|
|
992
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
1066
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && device_isFiniteNumber(candidate.bounds.x) && device_isFiniteNumber(candidate.bounds.y) && device_isFiniteNumber(candidate.bounds.width) && device_isFiniteNumber(candidate.bounds.height);
|
|
993
1067
|
}
|
|
994
1068
|
function readDarwinDisplayGeometries() {
|
|
995
1069
|
const bin = getDisplayInfoBinary();
|
|
@@ -1059,11 +1133,6 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1059
1133
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1060
1134
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1061
1135
|
}
|
|
1062
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1063
|
-
if (!displays.length) return;
|
|
1064
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1065
|
-
return displays.find((display)=>display.id === displayId);
|
|
1066
|
-
}
|
|
1067
1136
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1068
1137
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1069
1138
|
if ('win32' === process.platform) {
|
|
@@ -1156,13 +1225,24 @@ function normalizePrimaryKey(key) {
|
|
|
1156
1225
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1157
1226
|
}
|
|
1158
1227
|
class ComputerDevice {
|
|
1228
|
+
usesPhysicalWindowsCoordinates() {
|
|
1229
|
+
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1230
|
+
}
|
|
1231
|
+
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1232
|
+
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1233
|
+
assertLegacyWindowsCoordinateCompatibility({
|
|
1234
|
+
geometry: this.displayGeometry,
|
|
1235
|
+
screenshotBase64,
|
|
1236
|
+
inputSize: this.inputDriver.getScreenSize()
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1159
1239
|
async moveGlobalPointer(point, context, smooth) {
|
|
1160
1240
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1161
1241
|
const target = {
|
|
1162
1242
|
x: Math.round(point.x),
|
|
1163
1243
|
y: Math.round(point.y)
|
|
1164
1244
|
};
|
|
1165
|
-
if (
|
|
1245
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1166
1246
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1167
1247
|
smoothSteps: smooth?.smoothSteps,
|
|
1168
1248
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1214,7 +1294,7 @@ class ComputerDevice {
|
|
|
1214
1294
|
}
|
|
1215
1295
|
static async listDisplays() {
|
|
1216
1296
|
try {
|
|
1217
|
-
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1297
|
+
if ('win32' === process.platform) return listWindowsDisplays(discoverWindowsDisplays().geometries);
|
|
1218
1298
|
const displays = await external_screenshot_desktop_default().listDisplays();
|
|
1219
1299
|
return displays.map((d)=>({
|
|
1220
1300
|
id: String(d.id),
|
|
@@ -1252,7 +1332,10 @@ class ComputerDevice {
|
|
|
1252
1332
|
}
|
|
1253
1333
|
}
|
|
1254
1334
|
device_libnut = await getLibnut();
|
|
1255
|
-
const
|
|
1335
|
+
const windowsDisplayDiscovery = 'win32' === process.platform ? discoverWindowsDisplays() : void 0;
|
|
1336
|
+
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1337
|
+
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1338
|
+
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1256
1339
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1257
1340
|
const size = await this.size();
|
|
1258
1341
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1263,6 +1346,7 @@ Platform: ${process.platform}
|
|
|
1263
1346
|
Display: ${this.displayId || 'Primary'}
|
|
1264
1347
|
Screen Size: ${size.width}x${size.height}
|
|
1265
1348
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1349
|
+
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1266
1350
|
`;
|
|
1267
1351
|
debugDevice('Computer device connected', this.description);
|
|
1268
1352
|
await this.healthCheck(displays);
|
|
@@ -1286,7 +1370,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1286
1370
|
}
|
|
1287
1371
|
async healthCheck(displays) {
|
|
1288
1372
|
console.log('[HealthCheck] Starting health check...');
|
|
1289
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-
|
|
1373
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1-beta-20260922030100.0");
|
|
1290
1374
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1291
1375
|
const screenshotTimeout = 15000;
|
|
1292
1376
|
let timeoutId;
|
|
@@ -1298,10 +1382,12 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1298
1382
|
timeoutPromise
|
|
1299
1383
|
]);
|
|
1300
1384
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1385
|
+
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1301
1386
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1302
|
-
const
|
|
1387
|
+
const usePhysicalWindowsCoordinates = this.usesPhysicalWindowsCoordinates();
|
|
1388
|
+
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1303
1389
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1304
|
-
if (
|
|
1390
|
+
if (usePhysicalWindowsCoordinates) {
|
|
1305
1391
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1306
1392
|
const bounds = this.displayGeometry.bounds;
|
|
1307
1393
|
const target = {
|
|
@@ -1367,7 +1453,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1367
1453
|
debugDevice('Taking screenshot', {
|
|
1368
1454
|
displayId: this.displayId
|
|
1369
1455
|
});
|
|
1370
|
-
if (
|
|
1456
|
+
if (this.usesPhysicalWindowsCoordinates()) return this.screenshotViaPowershell();
|
|
1371
1457
|
const options = {
|
|
1372
1458
|
format: 'png'
|
|
1373
1459
|
};
|
|
@@ -1530,7 +1616,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1530
1616
|
this.inputDriver.sendKey(key, modifiers);
|
|
1531
1617
|
}
|
|
1532
1618
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1533
|
-
if (
|
|
1619
|
+
if (this.usesPhysicalWindowsCoordinates()) {
|
|
1534
1620
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1535
1621
|
if (activeWindowRect) {
|
|
1536
1622
|
const activeWindowCenter = {
|
|
@@ -1674,6 +1760,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1674
1760
|
device_define_property(this, "options", void 0);
|
|
1675
1761
|
device_define_property(this, "displayId", void 0);
|
|
1676
1762
|
device_define_property(this, "displayGeometry", void 0);
|
|
1763
|
+
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1677
1764
|
device_define_property(this, "description", void 0);
|
|
1678
1765
|
device_define_property(this, "destroyed", false);
|
|
1679
1766
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2820,7 +2907,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2820
2907
|
}
|
|
2821
2908
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2822
2909
|
function version() {
|
|
2823
|
-
const currentVersion = "1.13.1-beta-
|
|
2910
|
+
const currentVersion = "1.13.1-beta-20260922030100.0";
|
|
2824
2911
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2825
2912
|
return currentVersion;
|
|
2826
2913
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -70,6 +70,7 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
70
70
|
private options?;
|
|
71
71
|
private displayId?;
|
|
72
72
|
private displayGeometry?;
|
|
73
|
+
private windowsCoordinateMode;
|
|
73
74
|
private description?;
|
|
74
75
|
private destroyed;
|
|
75
76
|
private xvfbInstance?;
|
|
@@ -87,6 +88,8 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
87
88
|
uri?: string;
|
|
88
89
|
readonly inputPrimitives: ComputerInputPrimitives;
|
|
89
90
|
constructor(options?: ComputerDeviceOpt);
|
|
91
|
+
private usesPhysicalWindowsCoordinates;
|
|
92
|
+
private assertLegacyWindowsCoordinateCompatibility;
|
|
90
93
|
private moveGlobalPointer;
|
|
91
94
|
private moveDisplayPointer;
|
|
92
95
|
private performPointerDrag;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.13.1-beta-
|
|
3
|
+
"version": "1.13.1-beta-20260922030100.0",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
"@computer-use/libnut": "^4.2.0",
|
|
34
34
|
"clipboardy": "^4.0.0",
|
|
35
35
|
"screenshot-desktop": "^1.15.3",
|
|
36
|
-
"@midscene/core": "1.13.1-beta-
|
|
37
|
-
"@midscene/shared": "1.13.1-beta-
|
|
36
|
+
"@midscene/core": "1.13.1-beta-20260922030100.0",
|
|
37
|
+
"@midscene/shared": "1.13.1-beta-20260922030100.0"
|
|
38
38
|
},
|
|
39
39
|
"optionalDependencies": {
|
|
40
40
|
"node-mac-permissions": "2.5.0"
|