@midscene/computer 1.13.1-beta-20260922030100.0 → 1.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +57 -144
- package/dist/es/index.mjs +57 -144
- package/dist/lib/cli.js +57 -144
- package/dist/lib/index.js +57 -144
- package/dist/types/index.d.ts +0 -3
- package/package.json +4 -4
package/dist/es/cli.mjs
CHANGED
|
@@ -461,11 +461,17 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
461
461
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
462
462
|
}
|
|
463
463
|
`.trim();
|
|
464
|
-
function
|
|
464
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
465
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
466
|
+
$ErrorActionPreference = 'Stop'
|
|
467
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
468
|
+
${script}`;
|
|
469
|
+
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
465
470
|
return execFileSync('powershell.exe', [
|
|
466
471
|
'-NoProfile',
|
|
467
|
-
'-
|
|
468
|
-
|
|
472
|
+
'-NonInteractive',
|
|
473
|
+
'-EncodedCommand',
|
|
474
|
+
encoded
|
|
469
475
|
], {
|
|
470
476
|
encoding: 'utf8',
|
|
471
477
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -473,118 +479,6 @@ function runWindowsPowershellCommand(script) {
|
|
|
473
479
|
windowsHide: true
|
|
474
480
|
});
|
|
475
481
|
}
|
|
476
|
-
function runWindowsPowershell(script) {
|
|
477
|
-
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
478
|
-
$ErrorActionPreference = 'Stop'
|
|
479
|
-
${script}`);
|
|
480
|
-
}
|
|
481
|
-
function runWindowsPhysicalPixelPowershell(script) {
|
|
482
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
483
|
-
$ErrorActionPreference = 'Stop'
|
|
484
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
485
|
-
${script}`;
|
|
486
|
-
return runWindowsPowershellCommand(physicalPixelScript);
|
|
487
|
-
}
|
|
488
|
-
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
489
|
-
}
|
|
490
|
-
function isFiniteNumber(value) {
|
|
491
|
-
return 'number' == typeof value && Number.isFinite(value);
|
|
492
|
-
}
|
|
493
|
-
function isWindowsDisplayGeometry(value) {
|
|
494
|
-
if (!value || 'object' != typeof value) return false;
|
|
495
|
-
const candidate = value;
|
|
496
|
-
const bounds = candidate.bounds;
|
|
497
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
498
|
-
}
|
|
499
|
-
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
500
|
-
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
501
|
-
const script = `
|
|
502
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
503
|
-
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
504
|
-
$bounds = $_.Bounds
|
|
505
|
-
[PSCustomObject]@{
|
|
506
|
-
id = $_.DeviceName
|
|
507
|
-
name = $_.DeviceName
|
|
508
|
-
primary = $_.Primary
|
|
509
|
-
bounds = [PSCustomObject]@{
|
|
510
|
-
x = $bounds.X
|
|
511
|
-
y = $bounds.Y
|
|
512
|
-
width = $bounds.Width
|
|
513
|
-
height = $bounds.Height
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
})
|
|
517
|
-
${serializeScreens}
|
|
518
|
-
`.trim();
|
|
519
|
-
const output = runPowershell(script).trim();
|
|
520
|
-
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
521
|
-
const parsed = JSON.parse(output);
|
|
522
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
523
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
524
|
-
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
525
|
-
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
526
|
-
return displays;
|
|
527
|
-
}
|
|
528
|
-
function readWindowsDisplayGeometries() {
|
|
529
|
-
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
530
|
-
}
|
|
531
|
-
function discoverWindowsDisplays(runners = {
|
|
532
|
-
physical: runWindowsPhysicalPixelPowershell,
|
|
533
|
-
legacy: runWindowsPowershell
|
|
534
|
-
}) {
|
|
535
|
-
try {
|
|
536
|
-
return {
|
|
537
|
-
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
538
|
-
coordinateMode: 'physical'
|
|
539
|
-
};
|
|
540
|
-
} catch (error) {
|
|
541
|
-
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
542
|
-
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
543
|
-
return {
|
|
544
|
-
geometries,
|
|
545
|
-
coordinateMode: 'legacy'
|
|
546
|
-
};
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
function readPngDimensions(dataUri) {
|
|
550
|
-
const prefix = 'data:image/png;base64,';
|
|
551
|
-
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
552
|
-
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
553
|
-
const pngSignature = Buffer.from([
|
|
554
|
-
137,
|
|
555
|
-
80,
|
|
556
|
-
78,
|
|
557
|
-
71,
|
|
558
|
-
13,
|
|
559
|
-
10,
|
|
560
|
-
26,
|
|
561
|
-
10
|
|
562
|
-
]);
|
|
563
|
-
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
564
|
-
const width = buffer.readUInt32BE(16);
|
|
565
|
-
const height = buffer.readUInt32BE(20);
|
|
566
|
-
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
567
|
-
return {
|
|
568
|
-
width,
|
|
569
|
-
height
|
|
570
|
-
};
|
|
571
|
-
}
|
|
572
|
-
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
573
|
-
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
574
|
-
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
575
|
-
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
576
|
-
const displaySize = {
|
|
577
|
-
width: Math.round(geometry.bounds.width),
|
|
578
|
-
height: Math.round(geometry.bounds.height)
|
|
579
|
-
};
|
|
580
|
-
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
581
|
-
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
582
|
-
}
|
|
583
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
584
|
-
if (!displays.length) return;
|
|
585
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
586
|
-
return displays.find((display)=>display.id === displayId);
|
|
587
|
-
}
|
|
588
482
|
function windows_pointer_define_property(obj, key, value) {
|
|
589
483
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
590
484
|
value: value,
|
|
@@ -902,6 +796,28 @@ const EDGE_SCROLL_SPEC = {
|
|
|
902
796
|
function escapePowershellSingleQuoted(value) {
|
|
903
797
|
return value.replace(/'/g, "''");
|
|
904
798
|
}
|
|
799
|
+
function readWindowsDisplayGeometries() {
|
|
800
|
+
const script = `
|
|
801
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
802
|
+
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
803
|
+
$b = $_.Bounds
|
|
804
|
+
[PSCustomObject]@{
|
|
805
|
+
id = $_.DeviceName
|
|
806
|
+
name = $_.DeviceName
|
|
807
|
+
primary = $_.Primary
|
|
808
|
+
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
ConvertTo-Json @($s) -Compress
|
|
812
|
+
`.trim();
|
|
813
|
+
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
814
|
+
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
815
|
+
const parsed = JSON.parse(output);
|
|
816
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
817
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
818
|
+
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
819
|
+
return displays;
|
|
820
|
+
}
|
|
905
821
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
906
822
|
return geometries.map((display)=>({
|
|
907
823
|
id: display.id,
|
|
@@ -1005,13 +921,23 @@ function getDisplayInfoBinary() {
|
|
|
1005
921
|
displayInfoBinaryPath = binPath;
|
|
1006
922
|
return binPath;
|
|
1007
923
|
}
|
|
1008
|
-
function
|
|
924
|
+
function isFiniteNumber(value) {
|
|
1009
925
|
return 'number' == typeof value && Number.isFinite(value);
|
|
1010
926
|
}
|
|
927
|
+
function isDisplayBounds(value) {
|
|
928
|
+
if (!value || 'object' != typeof value) return false;
|
|
929
|
+
const bounds = value;
|
|
930
|
+
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
931
|
+
}
|
|
932
|
+
function isWindowsDisplayGeometry(value) {
|
|
933
|
+
if (!value || 'object' != typeof value) return false;
|
|
934
|
+
const candidate = value;
|
|
935
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
936
|
+
}
|
|
1011
937
|
function isDarwinDisplayGeometry(value) {
|
|
1012
938
|
if (!value || 'object' != typeof value) return false;
|
|
1013
939
|
const candidate = value;
|
|
1014
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
940
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
|
|
1015
941
|
}
|
|
1016
942
|
function readDarwinDisplayGeometries() {
|
|
1017
943
|
const bin = getDisplayInfoBinary();
|
|
@@ -1081,6 +1007,11 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1081
1007
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1082
1008
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1083
1009
|
}
|
|
1010
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1011
|
+
if (!displays.length) return;
|
|
1012
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1013
|
+
return displays.find((display)=>display.id === displayId);
|
|
1014
|
+
}
|
|
1084
1015
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1085
1016
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1086
1017
|
if ('win32' === process.platform) {
|
|
@@ -1173,24 +1104,13 @@ function normalizePrimaryKey(key) {
|
|
|
1173
1104
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1174
1105
|
}
|
|
1175
1106
|
class ComputerDevice {
|
|
1176
|
-
usesPhysicalWindowsCoordinates() {
|
|
1177
|
-
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1178
|
-
}
|
|
1179
|
-
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1180
|
-
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1181
|
-
assertLegacyWindowsCoordinateCompatibility({
|
|
1182
|
-
geometry: this.displayGeometry,
|
|
1183
|
-
screenshotBase64,
|
|
1184
|
-
inputSize: this.inputDriver.getScreenSize()
|
|
1185
|
-
});
|
|
1186
|
-
}
|
|
1187
1107
|
async moveGlobalPointer(point, context, smooth) {
|
|
1188
1108
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1189
1109
|
const target = {
|
|
1190
1110
|
x: Math.round(point.x),
|
|
1191
1111
|
y: Math.round(point.y)
|
|
1192
1112
|
};
|
|
1193
|
-
if (
|
|
1113
|
+
if ('win32' === process.platform) {
|
|
1194
1114
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1195
1115
|
smoothSteps: smooth?.smoothSteps,
|
|
1196
1116
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1242,7 +1162,7 @@ class ComputerDevice {
|
|
|
1242
1162
|
}
|
|
1243
1163
|
static async listDisplays() {
|
|
1244
1164
|
try {
|
|
1245
|
-
if ('win32' === process.platform) return listWindowsDisplays(
|
|
1165
|
+
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1246
1166
|
const displays = await screenshot_desktop.listDisplays();
|
|
1247
1167
|
return displays.map((d)=>({
|
|
1248
1168
|
id: String(d.id),
|
|
@@ -1280,10 +1200,7 @@ class ComputerDevice {
|
|
|
1280
1200
|
}
|
|
1281
1201
|
}
|
|
1282
1202
|
device_libnut = await getLibnut();
|
|
1283
|
-
const
|
|
1284
|
-
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1285
|
-
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1286
|
-
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1203
|
+
const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
|
|
1287
1204
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1288
1205
|
const size = await this.size();
|
|
1289
1206
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1294,7 +1211,6 @@ Platform: ${process.platform}
|
|
|
1294
1211
|
Display: ${this.displayId || 'Primary'}
|
|
1295
1212
|
Screen Size: ${size.width}x${size.height}
|
|
1296
1213
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1297
|
-
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1298
1214
|
`;
|
|
1299
1215
|
debugDevice('Computer device connected', this.description);
|
|
1300
1216
|
await this.healthCheck(displays);
|
|
@@ -1318,7 +1234,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1318
1234
|
}
|
|
1319
1235
|
async healthCheck(displays) {
|
|
1320
1236
|
console.log('[HealthCheck] Starting health check...');
|
|
1321
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1
|
|
1237
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1");
|
|
1322
1238
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1323
1239
|
const screenshotTimeout = 15000;
|
|
1324
1240
|
let timeoutId;
|
|
@@ -1330,12 +1246,10 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1330
1246
|
timeoutPromise
|
|
1331
1247
|
]);
|
|
1332
1248
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1333
|
-
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1334
1249
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1335
|
-
const
|
|
1336
|
-
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1250
|
+
const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1337
1251
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1338
|
-
if (
|
|
1252
|
+
if ('win32' === process.platform) {
|
|
1339
1253
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1340
1254
|
const bounds = this.displayGeometry.bounds;
|
|
1341
1255
|
const target = {
|
|
@@ -1401,7 +1315,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1401
1315
|
debugDevice('Taking screenshot', {
|
|
1402
1316
|
displayId: this.displayId
|
|
1403
1317
|
});
|
|
1404
|
-
if (
|
|
1318
|
+
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
1405
1319
|
const options = {
|
|
1406
1320
|
format: 'png'
|
|
1407
1321
|
};
|
|
@@ -1564,7 +1478,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1564
1478
|
this.inputDriver.sendKey(key, modifiers);
|
|
1565
1479
|
}
|
|
1566
1480
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1567
|
-
if (
|
|
1481
|
+
if ('win32' === process.platform) {
|
|
1568
1482
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1569
1483
|
if (activeWindowRect) {
|
|
1570
1484
|
const activeWindowCenter = {
|
|
@@ -1708,7 +1622,6 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1708
1622
|
device_define_property(this, "options", void 0);
|
|
1709
1623
|
device_define_property(this, "displayId", void 0);
|
|
1710
1624
|
device_define_property(this, "displayGeometry", void 0);
|
|
1711
|
-
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1712
1625
|
device_define_property(this, "description", void 0);
|
|
1713
1626
|
device_define_property(this, "destroyed", false);
|
|
1714
1627
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2810,7 +2723,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2810
2723
|
});
|
|
2811
2724
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2812
2725
|
stripPrefix: 'computer_',
|
|
2813
|
-
version: "1.13.1
|
|
2726
|
+
version: "1.13.1",
|
|
2814
2727
|
extraCommands: createReportCliCommands()
|
|
2815
2728
|
}).catch((e)=>{
|
|
2816
2729
|
process.exit(reportCLIError(e));
|
package/dist/es/index.mjs
CHANGED
|
@@ -461,11 +461,17 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
461
461
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
462
462
|
}
|
|
463
463
|
`.trim();
|
|
464
|
-
function
|
|
464
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
465
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
466
|
+
$ErrorActionPreference = 'Stop'
|
|
467
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
468
|
+
${script}`;
|
|
469
|
+
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
465
470
|
return execFileSync('powershell.exe', [
|
|
466
471
|
'-NoProfile',
|
|
467
|
-
'-
|
|
468
|
-
|
|
472
|
+
'-NonInteractive',
|
|
473
|
+
'-EncodedCommand',
|
|
474
|
+
encoded
|
|
469
475
|
], {
|
|
470
476
|
encoding: 'utf8',
|
|
471
477
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -473,118 +479,6 @@ function runWindowsPowershellCommand(script) {
|
|
|
473
479
|
windowsHide: true
|
|
474
480
|
});
|
|
475
481
|
}
|
|
476
|
-
function runWindowsPowershell(script) {
|
|
477
|
-
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
478
|
-
$ErrorActionPreference = 'Stop'
|
|
479
|
-
${script}`);
|
|
480
|
-
}
|
|
481
|
-
function runWindowsPhysicalPixelPowershell(script) {
|
|
482
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
483
|
-
$ErrorActionPreference = 'Stop'
|
|
484
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
485
|
-
${script}`;
|
|
486
|
-
return runWindowsPowershellCommand(physicalPixelScript);
|
|
487
|
-
}
|
|
488
|
-
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
489
|
-
}
|
|
490
|
-
function isFiniteNumber(value) {
|
|
491
|
-
return 'number' == typeof value && Number.isFinite(value);
|
|
492
|
-
}
|
|
493
|
-
function isWindowsDisplayGeometry(value) {
|
|
494
|
-
if (!value || 'object' != typeof value) return false;
|
|
495
|
-
const candidate = value;
|
|
496
|
-
const bounds = candidate.bounds;
|
|
497
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
498
|
-
}
|
|
499
|
-
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
500
|
-
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
501
|
-
const script = `
|
|
502
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
503
|
-
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
504
|
-
$bounds = $_.Bounds
|
|
505
|
-
[PSCustomObject]@{
|
|
506
|
-
id = $_.DeviceName
|
|
507
|
-
name = $_.DeviceName
|
|
508
|
-
primary = $_.Primary
|
|
509
|
-
bounds = [PSCustomObject]@{
|
|
510
|
-
x = $bounds.X
|
|
511
|
-
y = $bounds.Y
|
|
512
|
-
width = $bounds.Width
|
|
513
|
-
height = $bounds.Height
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
})
|
|
517
|
-
${serializeScreens}
|
|
518
|
-
`.trim();
|
|
519
|
-
const output = runPowershell(script).trim();
|
|
520
|
-
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
521
|
-
const parsed = JSON.parse(output);
|
|
522
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
523
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
524
|
-
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
525
|
-
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
526
|
-
return displays;
|
|
527
|
-
}
|
|
528
|
-
function readWindowsDisplayGeometries() {
|
|
529
|
-
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
530
|
-
}
|
|
531
|
-
function discoverWindowsDisplays(runners = {
|
|
532
|
-
physical: runWindowsPhysicalPixelPowershell,
|
|
533
|
-
legacy: runWindowsPowershell
|
|
534
|
-
}) {
|
|
535
|
-
try {
|
|
536
|
-
return {
|
|
537
|
-
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
538
|
-
coordinateMode: 'physical'
|
|
539
|
-
};
|
|
540
|
-
} catch (error) {
|
|
541
|
-
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
542
|
-
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
543
|
-
return {
|
|
544
|
-
geometries,
|
|
545
|
-
coordinateMode: 'legacy'
|
|
546
|
-
};
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
function readPngDimensions(dataUri) {
|
|
550
|
-
const prefix = 'data:image/png;base64,';
|
|
551
|
-
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
552
|
-
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
553
|
-
const pngSignature = Buffer.from([
|
|
554
|
-
137,
|
|
555
|
-
80,
|
|
556
|
-
78,
|
|
557
|
-
71,
|
|
558
|
-
13,
|
|
559
|
-
10,
|
|
560
|
-
26,
|
|
561
|
-
10
|
|
562
|
-
]);
|
|
563
|
-
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
564
|
-
const width = buffer.readUInt32BE(16);
|
|
565
|
-
const height = buffer.readUInt32BE(20);
|
|
566
|
-
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
567
|
-
return {
|
|
568
|
-
width,
|
|
569
|
-
height
|
|
570
|
-
};
|
|
571
|
-
}
|
|
572
|
-
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
573
|
-
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
574
|
-
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
575
|
-
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
576
|
-
const displaySize = {
|
|
577
|
-
width: Math.round(geometry.bounds.width),
|
|
578
|
-
height: Math.round(geometry.bounds.height)
|
|
579
|
-
};
|
|
580
|
-
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
581
|
-
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
582
|
-
}
|
|
583
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
584
|
-
if (!displays.length) return;
|
|
585
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
586
|
-
return displays.find((display)=>display.id === displayId);
|
|
587
|
-
}
|
|
588
482
|
function windows_pointer_define_property(obj, key, value) {
|
|
589
483
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
590
484
|
value: value,
|
|
@@ -902,6 +796,28 @@ const EDGE_SCROLL_SPEC = {
|
|
|
902
796
|
function escapePowershellSingleQuoted(value) {
|
|
903
797
|
return value.replace(/'/g, "''");
|
|
904
798
|
}
|
|
799
|
+
function readWindowsDisplayGeometries() {
|
|
800
|
+
const script = `
|
|
801
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
802
|
+
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
803
|
+
$b = $_.Bounds
|
|
804
|
+
[PSCustomObject]@{
|
|
805
|
+
id = $_.DeviceName
|
|
806
|
+
name = $_.DeviceName
|
|
807
|
+
primary = $_.Primary
|
|
808
|
+
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
ConvertTo-Json @($s) -Compress
|
|
812
|
+
`.trim();
|
|
813
|
+
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
814
|
+
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
815
|
+
const parsed = JSON.parse(output);
|
|
816
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
817
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
818
|
+
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
819
|
+
return displays;
|
|
820
|
+
}
|
|
905
821
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
906
822
|
return geometries.map((display)=>({
|
|
907
823
|
id: display.id,
|
|
@@ -1005,13 +921,23 @@ function getDisplayInfoBinary() {
|
|
|
1005
921
|
displayInfoBinaryPath = binPath;
|
|
1006
922
|
return binPath;
|
|
1007
923
|
}
|
|
1008
|
-
function
|
|
924
|
+
function isFiniteNumber(value) {
|
|
1009
925
|
return 'number' == typeof value && Number.isFinite(value);
|
|
1010
926
|
}
|
|
927
|
+
function isDisplayBounds(value) {
|
|
928
|
+
if (!value || 'object' != typeof value) return false;
|
|
929
|
+
const bounds = value;
|
|
930
|
+
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
931
|
+
}
|
|
932
|
+
function isWindowsDisplayGeometry(value) {
|
|
933
|
+
if (!value || 'object' != typeof value) return false;
|
|
934
|
+
const candidate = value;
|
|
935
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
936
|
+
}
|
|
1011
937
|
function isDarwinDisplayGeometry(value) {
|
|
1012
938
|
if (!value || 'object' != typeof value) return false;
|
|
1013
939
|
const candidate = value;
|
|
1014
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
940
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
|
|
1015
941
|
}
|
|
1016
942
|
function readDarwinDisplayGeometries() {
|
|
1017
943
|
const bin = getDisplayInfoBinary();
|
|
@@ -1081,6 +1007,11 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1081
1007
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1082
1008
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1083
1009
|
}
|
|
1010
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1011
|
+
if (!displays.length) return;
|
|
1012
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1013
|
+
return displays.find((display)=>display.id === displayId);
|
|
1014
|
+
}
|
|
1084
1015
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1085
1016
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1086
1017
|
if ('win32' === process.platform) {
|
|
@@ -1173,24 +1104,13 @@ function normalizePrimaryKey(key) {
|
|
|
1173
1104
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1174
1105
|
}
|
|
1175
1106
|
class ComputerDevice {
|
|
1176
|
-
usesPhysicalWindowsCoordinates() {
|
|
1177
|
-
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1178
|
-
}
|
|
1179
|
-
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1180
|
-
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1181
|
-
assertLegacyWindowsCoordinateCompatibility({
|
|
1182
|
-
geometry: this.displayGeometry,
|
|
1183
|
-
screenshotBase64,
|
|
1184
|
-
inputSize: this.inputDriver.getScreenSize()
|
|
1185
|
-
});
|
|
1186
|
-
}
|
|
1187
1107
|
async moveGlobalPointer(point, context, smooth) {
|
|
1188
1108
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1189
1109
|
const target = {
|
|
1190
1110
|
x: Math.round(point.x),
|
|
1191
1111
|
y: Math.round(point.y)
|
|
1192
1112
|
};
|
|
1193
|
-
if (
|
|
1113
|
+
if ('win32' === process.platform) {
|
|
1194
1114
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1195
1115
|
smoothSteps: smooth?.smoothSteps,
|
|
1196
1116
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1242,7 +1162,7 @@ class ComputerDevice {
|
|
|
1242
1162
|
}
|
|
1243
1163
|
static async listDisplays() {
|
|
1244
1164
|
try {
|
|
1245
|
-
if ('win32' === process.platform) return listWindowsDisplays(
|
|
1165
|
+
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1246
1166
|
const displays = await screenshot_desktop.listDisplays();
|
|
1247
1167
|
return displays.map((d)=>({
|
|
1248
1168
|
id: String(d.id),
|
|
@@ -1280,10 +1200,7 @@ class ComputerDevice {
|
|
|
1280
1200
|
}
|
|
1281
1201
|
}
|
|
1282
1202
|
device_libnut = await getLibnut();
|
|
1283
|
-
const
|
|
1284
|
-
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1285
|
-
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1286
|
-
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1203
|
+
const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
|
|
1287
1204
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1288
1205
|
const size = await this.size();
|
|
1289
1206
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1294,7 +1211,6 @@ Platform: ${process.platform}
|
|
|
1294
1211
|
Display: ${this.displayId || 'Primary'}
|
|
1295
1212
|
Screen Size: ${size.width}x${size.height}
|
|
1296
1213
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1297
|
-
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1298
1214
|
`;
|
|
1299
1215
|
debugDevice('Computer device connected', this.description);
|
|
1300
1216
|
await this.healthCheck(displays);
|
|
@@ -1318,7 +1234,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1318
1234
|
}
|
|
1319
1235
|
async healthCheck(displays) {
|
|
1320
1236
|
console.log('[HealthCheck] Starting health check...');
|
|
1321
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1
|
|
1237
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1");
|
|
1322
1238
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1323
1239
|
const screenshotTimeout = 15000;
|
|
1324
1240
|
let timeoutId;
|
|
@@ -1330,12 +1246,10 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1330
1246
|
timeoutPromise
|
|
1331
1247
|
]);
|
|
1332
1248
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1333
|
-
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1334
1249
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1335
|
-
const
|
|
1336
|
-
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1250
|
+
const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1337
1251
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1338
|
-
if (
|
|
1252
|
+
if ('win32' === process.platform) {
|
|
1339
1253
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1340
1254
|
const bounds = this.displayGeometry.bounds;
|
|
1341
1255
|
const target = {
|
|
@@ -1401,7 +1315,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1401
1315
|
debugDevice('Taking screenshot', {
|
|
1402
1316
|
displayId: this.displayId
|
|
1403
1317
|
});
|
|
1404
|
-
if (
|
|
1318
|
+
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
1405
1319
|
const options = {
|
|
1406
1320
|
format: 'png'
|
|
1407
1321
|
};
|
|
@@ -1564,7 +1478,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1564
1478
|
this.inputDriver.sendKey(key, modifiers);
|
|
1565
1479
|
}
|
|
1566
1480
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1567
|
-
if (
|
|
1481
|
+
if ('win32' === process.platform) {
|
|
1568
1482
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1569
1483
|
if (activeWindowRect) {
|
|
1570
1484
|
const activeWindowCenter = {
|
|
@@ -1708,7 +1622,6 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1708
1622
|
device_define_property(this, "options", void 0);
|
|
1709
1623
|
device_define_property(this, "displayId", void 0);
|
|
1710
1624
|
device_define_property(this, "displayGeometry", void 0);
|
|
1711
|
-
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1712
1625
|
device_define_property(this, "description", void 0);
|
|
1713
1626
|
device_define_property(this, "destroyed", false);
|
|
1714
1627
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2849,7 +2762,7 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2849
2762
|
}
|
|
2850
2763
|
}
|
|
2851
2764
|
function version() {
|
|
2852
|
-
const currentVersion = "1.13.1
|
|
2765
|
+
const currentVersion = "1.13.1";
|
|
2853
2766
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2854
2767
|
return currentVersion;
|
|
2855
2768
|
}
|
package/dist/lib/cli.js
CHANGED
|
@@ -486,11 +486,17 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
486
486
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
487
487
|
}
|
|
488
488
|
`.trim();
|
|
489
|
-
function
|
|
489
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
490
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
491
|
+
$ErrorActionPreference = 'Stop'
|
|
492
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
493
|
+
${script}`;
|
|
494
|
+
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
490
495
|
return (0, external_node_child_process_namespaceObject.execFileSync)('powershell.exe', [
|
|
491
496
|
'-NoProfile',
|
|
492
|
-
'-
|
|
493
|
-
|
|
497
|
+
'-NonInteractive',
|
|
498
|
+
'-EncodedCommand',
|
|
499
|
+
encoded
|
|
494
500
|
], {
|
|
495
501
|
encoding: 'utf8',
|
|
496
502
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -498,118 +504,6 @@ function runWindowsPowershellCommand(script) {
|
|
|
498
504
|
windowsHide: true
|
|
499
505
|
});
|
|
500
506
|
}
|
|
501
|
-
function runWindowsPowershell(script) {
|
|
502
|
-
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
503
|
-
$ErrorActionPreference = 'Stop'
|
|
504
|
-
${script}`);
|
|
505
|
-
}
|
|
506
|
-
function runWindowsPhysicalPixelPowershell(script) {
|
|
507
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
508
|
-
$ErrorActionPreference = 'Stop'
|
|
509
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
510
|
-
${script}`;
|
|
511
|
-
return runWindowsPowershellCommand(physicalPixelScript);
|
|
512
|
-
}
|
|
513
|
-
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
514
|
-
}
|
|
515
|
-
function isFiniteNumber(value) {
|
|
516
|
-
return 'number' == typeof value && Number.isFinite(value);
|
|
517
|
-
}
|
|
518
|
-
function isWindowsDisplayGeometry(value) {
|
|
519
|
-
if (!value || 'object' != typeof value) return false;
|
|
520
|
-
const candidate = value;
|
|
521
|
-
const bounds = candidate.bounds;
|
|
522
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
523
|
-
}
|
|
524
|
-
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
525
|
-
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
526
|
-
const script = `
|
|
527
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
528
|
-
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
529
|
-
$bounds = $_.Bounds
|
|
530
|
-
[PSCustomObject]@{
|
|
531
|
-
id = $_.DeviceName
|
|
532
|
-
name = $_.DeviceName
|
|
533
|
-
primary = $_.Primary
|
|
534
|
-
bounds = [PSCustomObject]@{
|
|
535
|
-
x = $bounds.X
|
|
536
|
-
y = $bounds.Y
|
|
537
|
-
width = $bounds.Width
|
|
538
|
-
height = $bounds.Height
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
})
|
|
542
|
-
${serializeScreens}
|
|
543
|
-
`.trim();
|
|
544
|
-
const output = runPowershell(script).trim();
|
|
545
|
-
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
546
|
-
const parsed = JSON.parse(output);
|
|
547
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
548
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
549
|
-
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
550
|
-
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
551
|
-
return displays;
|
|
552
|
-
}
|
|
553
|
-
function readWindowsDisplayGeometries() {
|
|
554
|
-
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
555
|
-
}
|
|
556
|
-
function discoverWindowsDisplays(runners = {
|
|
557
|
-
physical: runWindowsPhysicalPixelPowershell,
|
|
558
|
-
legacy: runWindowsPowershell
|
|
559
|
-
}) {
|
|
560
|
-
try {
|
|
561
|
-
return {
|
|
562
|
-
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
563
|
-
coordinateMode: 'physical'
|
|
564
|
-
};
|
|
565
|
-
} catch (error) {
|
|
566
|
-
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
567
|
-
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
568
|
-
return {
|
|
569
|
-
geometries,
|
|
570
|
-
coordinateMode: 'legacy'
|
|
571
|
-
};
|
|
572
|
-
}
|
|
573
|
-
}
|
|
574
|
-
function readPngDimensions(dataUri) {
|
|
575
|
-
const prefix = 'data:image/png;base64,';
|
|
576
|
-
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
577
|
-
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
578
|
-
const pngSignature = Buffer.from([
|
|
579
|
-
137,
|
|
580
|
-
80,
|
|
581
|
-
78,
|
|
582
|
-
71,
|
|
583
|
-
13,
|
|
584
|
-
10,
|
|
585
|
-
26,
|
|
586
|
-
10
|
|
587
|
-
]);
|
|
588
|
-
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
589
|
-
const width = buffer.readUInt32BE(16);
|
|
590
|
-
const height = buffer.readUInt32BE(20);
|
|
591
|
-
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
592
|
-
return {
|
|
593
|
-
width,
|
|
594
|
-
height
|
|
595
|
-
};
|
|
596
|
-
}
|
|
597
|
-
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
598
|
-
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
599
|
-
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
600
|
-
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
601
|
-
const displaySize = {
|
|
602
|
-
width: Math.round(geometry.bounds.width),
|
|
603
|
-
height: Math.round(geometry.bounds.height)
|
|
604
|
-
};
|
|
605
|
-
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
606
|
-
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
607
|
-
}
|
|
608
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
609
|
-
if (!displays.length) return;
|
|
610
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
611
|
-
return displays.find((display)=>display.id === displayId);
|
|
612
|
-
}
|
|
613
507
|
function windows_pointer_define_property(obj, key, value) {
|
|
614
508
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
615
509
|
value: value,
|
|
@@ -928,6 +822,28 @@ const EDGE_SCROLL_SPEC = {
|
|
|
928
822
|
function escapePowershellSingleQuoted(value) {
|
|
929
823
|
return value.replace(/'/g, "''");
|
|
930
824
|
}
|
|
825
|
+
function readWindowsDisplayGeometries() {
|
|
826
|
+
const script = `
|
|
827
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
828
|
+
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
829
|
+
$b = $_.Bounds
|
|
830
|
+
[PSCustomObject]@{
|
|
831
|
+
id = $_.DeviceName
|
|
832
|
+
name = $_.DeviceName
|
|
833
|
+
primary = $_.Primary
|
|
834
|
+
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
ConvertTo-Json @($s) -Compress
|
|
838
|
+
`.trim();
|
|
839
|
+
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
840
|
+
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
841
|
+
const parsed = JSON.parse(output);
|
|
842
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
843
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
844
|
+
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
845
|
+
return displays;
|
|
846
|
+
}
|
|
931
847
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
932
848
|
return geometries.map((display)=>({
|
|
933
849
|
id: display.id,
|
|
@@ -1031,13 +947,23 @@ function getDisplayInfoBinary() {
|
|
|
1031
947
|
displayInfoBinaryPath = binPath;
|
|
1032
948
|
return binPath;
|
|
1033
949
|
}
|
|
1034
|
-
function
|
|
950
|
+
function isFiniteNumber(value) {
|
|
1035
951
|
return 'number' == typeof value && Number.isFinite(value);
|
|
1036
952
|
}
|
|
953
|
+
function isDisplayBounds(value) {
|
|
954
|
+
if (!value || 'object' != typeof value) return false;
|
|
955
|
+
const bounds = value;
|
|
956
|
+
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
957
|
+
}
|
|
958
|
+
function isWindowsDisplayGeometry(value) {
|
|
959
|
+
if (!value || 'object' != typeof value) return false;
|
|
960
|
+
const candidate = value;
|
|
961
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
962
|
+
}
|
|
1037
963
|
function isDarwinDisplayGeometry(value) {
|
|
1038
964
|
if (!value || 'object' != typeof value) return false;
|
|
1039
965
|
const candidate = value;
|
|
1040
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
966
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
|
|
1041
967
|
}
|
|
1042
968
|
function readDarwinDisplayGeometries() {
|
|
1043
969
|
const bin = getDisplayInfoBinary();
|
|
@@ -1107,6 +1033,11 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1107
1033
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1108
1034
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1109
1035
|
}
|
|
1036
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1037
|
+
if (!displays.length) return;
|
|
1038
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1039
|
+
return displays.find((display)=>display.id === displayId);
|
|
1040
|
+
}
|
|
1110
1041
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1111
1042
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1112
1043
|
if ('win32' === process.platform) {
|
|
@@ -1199,24 +1130,13 @@ function normalizePrimaryKey(key) {
|
|
|
1199
1130
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1200
1131
|
}
|
|
1201
1132
|
class ComputerDevice {
|
|
1202
|
-
usesPhysicalWindowsCoordinates() {
|
|
1203
|
-
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1204
|
-
}
|
|
1205
|
-
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1206
|
-
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1207
|
-
assertLegacyWindowsCoordinateCompatibility({
|
|
1208
|
-
geometry: this.displayGeometry,
|
|
1209
|
-
screenshotBase64,
|
|
1210
|
-
inputSize: this.inputDriver.getScreenSize()
|
|
1211
|
-
});
|
|
1212
|
-
}
|
|
1213
1133
|
async moveGlobalPointer(point, context, smooth) {
|
|
1214
1134
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1215
1135
|
const target = {
|
|
1216
1136
|
x: Math.round(point.x),
|
|
1217
1137
|
y: Math.round(point.y)
|
|
1218
1138
|
};
|
|
1219
|
-
if (
|
|
1139
|
+
if ('win32' === process.platform) {
|
|
1220
1140
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1221
1141
|
smoothSteps: smooth?.smoothSteps,
|
|
1222
1142
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1268,7 +1188,7 @@ class ComputerDevice {
|
|
|
1268
1188
|
}
|
|
1269
1189
|
static async listDisplays() {
|
|
1270
1190
|
try {
|
|
1271
|
-
if ('win32' === process.platform) return listWindowsDisplays(
|
|
1191
|
+
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1272
1192
|
const displays = await external_screenshot_desktop_default().listDisplays();
|
|
1273
1193
|
return displays.map((d)=>({
|
|
1274
1194
|
id: String(d.id),
|
|
@@ -1306,10 +1226,7 @@ class ComputerDevice {
|
|
|
1306
1226
|
}
|
|
1307
1227
|
}
|
|
1308
1228
|
device_libnut = await getLibnut();
|
|
1309
|
-
const
|
|
1310
|
-
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1311
|
-
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1312
|
-
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1229
|
+
const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
|
|
1313
1230
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1314
1231
|
const size = await this.size();
|
|
1315
1232
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1320,7 +1237,6 @@ Platform: ${process.platform}
|
|
|
1320
1237
|
Display: ${this.displayId || 'Primary'}
|
|
1321
1238
|
Screen Size: ${size.width}x${size.height}
|
|
1322
1239
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1323
|
-
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1324
1240
|
`;
|
|
1325
1241
|
debugDevice('Computer device connected', this.description);
|
|
1326
1242
|
await this.healthCheck(displays);
|
|
@@ -1344,7 +1260,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1344
1260
|
}
|
|
1345
1261
|
async healthCheck(displays) {
|
|
1346
1262
|
console.log('[HealthCheck] Starting health check...');
|
|
1347
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1
|
|
1263
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1");
|
|
1348
1264
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1349
1265
|
const screenshotTimeout = 15000;
|
|
1350
1266
|
let timeoutId;
|
|
@@ -1356,12 +1272,10 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1356
1272
|
timeoutPromise
|
|
1357
1273
|
]);
|
|
1358
1274
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1359
|
-
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1360
1275
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1361
|
-
const
|
|
1362
|
-
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1276
|
+
const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1363
1277
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1364
|
-
if (
|
|
1278
|
+
if ('win32' === process.platform) {
|
|
1365
1279
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1366
1280
|
const bounds = this.displayGeometry.bounds;
|
|
1367
1281
|
const target = {
|
|
@@ -1427,7 +1341,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1427
1341
|
debugDevice('Taking screenshot', {
|
|
1428
1342
|
displayId: this.displayId
|
|
1429
1343
|
});
|
|
1430
|
-
if (
|
|
1344
|
+
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
1431
1345
|
const options = {
|
|
1432
1346
|
format: 'png'
|
|
1433
1347
|
};
|
|
@@ -1590,7 +1504,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1590
1504
|
this.inputDriver.sendKey(key, modifiers);
|
|
1591
1505
|
}
|
|
1592
1506
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1593
|
-
if (
|
|
1507
|
+
if ('win32' === process.platform) {
|
|
1594
1508
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1595
1509
|
if (activeWindowRect) {
|
|
1596
1510
|
const activeWindowCenter = {
|
|
@@ -1734,7 +1648,6 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1734
1648
|
device_define_property(this, "options", void 0);
|
|
1735
1649
|
device_define_property(this, "displayId", void 0);
|
|
1736
1650
|
device_define_property(this, "displayGeometry", void 0);
|
|
1737
|
-
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1738
1651
|
device_define_property(this, "description", void 0);
|
|
1739
1652
|
device_define_property(this, "destroyed", false);
|
|
1740
1653
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2837,7 +2750,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2837
2750
|
});
|
|
2838
2751
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
2839
2752
|
stripPrefix: 'computer_',
|
|
2840
|
-
version: "1.13.1
|
|
2753
|
+
version: "1.13.1",
|
|
2841
2754
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
2842
2755
|
}).catch((e)=>{
|
|
2843
2756
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|
package/dist/lib/index.js
CHANGED
|
@@ -512,11 +512,17 @@ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
|
512
512
|
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
513
513
|
}
|
|
514
514
|
`.trim();
|
|
515
|
-
function
|
|
515
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
516
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
517
|
+
$ErrorActionPreference = 'Stop'
|
|
518
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
519
|
+
${script}`;
|
|
520
|
+
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
516
521
|
return (0, external_node_child_process_namespaceObject.execFileSync)('powershell.exe', [
|
|
517
522
|
'-NoProfile',
|
|
518
|
-
'-
|
|
519
|
-
|
|
523
|
+
'-NonInteractive',
|
|
524
|
+
'-EncodedCommand',
|
|
525
|
+
encoded
|
|
520
526
|
], {
|
|
521
527
|
encoding: 'utf8',
|
|
522
528
|
timeout: POWERSHELL_TIMEOUT_MS,
|
|
@@ -524,118 +530,6 @@ function runWindowsPowershellCommand(script) {
|
|
|
524
530
|
windowsHide: true
|
|
525
531
|
});
|
|
526
532
|
}
|
|
527
|
-
function runWindowsPowershell(script) {
|
|
528
|
-
return runWindowsPowershellCommand(`$ProgressPreference = 'SilentlyContinue'
|
|
529
|
-
$ErrorActionPreference = 'Stop'
|
|
530
|
-
${script}`);
|
|
531
|
-
}
|
|
532
|
-
function runWindowsPhysicalPixelPowershell(script) {
|
|
533
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
534
|
-
$ErrorActionPreference = 'Stop'
|
|
535
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
536
|
-
${script}`;
|
|
537
|
-
return runWindowsPowershellCommand(physicalPixelScript);
|
|
538
|
-
}
|
|
539
|
-
class WindowsDisplayEnumerationEmptyError extends Error {
|
|
540
|
-
}
|
|
541
|
-
function isFiniteNumber(value) {
|
|
542
|
-
return 'number' == typeof value && Number.isFinite(value);
|
|
543
|
-
}
|
|
544
|
-
function isWindowsDisplayGeometry(value) {
|
|
545
|
-
if (!value || 'object' != typeof value) return false;
|
|
546
|
-
const candidate = value;
|
|
547
|
-
const bounds = candidate.bounds;
|
|
548
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
549
|
-
}
|
|
550
|
-
function readWindowsDisplayGeometriesWith(runPowershell, context) {
|
|
551
|
-
const serializeScreens = 'legacy' === context ? 'ConvertTo-Json -InputObject $screens -Compress' : '[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))';
|
|
552
|
-
const script = `
|
|
553
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
554
|
-
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
555
|
-
$bounds = $_.Bounds
|
|
556
|
-
[PSCustomObject]@{
|
|
557
|
-
id = $_.DeviceName
|
|
558
|
-
name = $_.DeviceName
|
|
559
|
-
primary = $_.Primary
|
|
560
|
-
bounds = [PSCustomObject]@{
|
|
561
|
-
x = $bounds.X
|
|
562
|
-
y = $bounds.Y
|
|
563
|
-
width = $bounds.Width
|
|
564
|
-
height = $bounds.Height
|
|
565
|
-
}
|
|
566
|
-
}
|
|
567
|
-
})
|
|
568
|
-
${serializeScreens}
|
|
569
|
-
`.trim();
|
|
570
|
-
const output = runPowershell(script).trim();
|
|
571
|
-
if (!output) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no data`);
|
|
572
|
-
const parsed = JSON.parse(output);
|
|
573
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
574
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
575
|
-
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
576
|
-
if (0 === displays.length) throw new WindowsDisplayEnumerationEmptyError(`Windows ${context} display enumeration returned no displays`);
|
|
577
|
-
return displays;
|
|
578
|
-
}
|
|
579
|
-
function readWindowsDisplayGeometries() {
|
|
580
|
-
return readWindowsDisplayGeometriesWith(runWindowsPhysicalPixelPowershell, 'physical');
|
|
581
|
-
}
|
|
582
|
-
function discoverWindowsDisplays(runners = {
|
|
583
|
-
physical: runWindowsPhysicalPixelPowershell,
|
|
584
|
-
legacy: runWindowsPowershell
|
|
585
|
-
}) {
|
|
586
|
-
try {
|
|
587
|
-
return {
|
|
588
|
-
geometries: readWindowsDisplayGeometriesWith(runners.physical, 'physical'),
|
|
589
|
-
coordinateMode: 'physical'
|
|
590
|
-
};
|
|
591
|
-
} catch (error) {
|
|
592
|
-
if (!(error instanceof WindowsDisplayEnumerationEmptyError)) throw error;
|
|
593
|
-
const geometries = readWindowsDisplayGeometriesWith(runners.legacy, 'legacy');
|
|
594
|
-
return {
|
|
595
|
-
geometries,
|
|
596
|
-
coordinateMode: 'legacy'
|
|
597
|
-
};
|
|
598
|
-
}
|
|
599
|
-
}
|
|
600
|
-
function readPngDimensions(dataUri) {
|
|
601
|
-
const prefix = 'data:image/png;base64,';
|
|
602
|
-
if (!dataUri.startsWith(prefix)) throw new Error('Screenshot is not a PNG data URI');
|
|
603
|
-
const buffer = Buffer.from(dataUri.slice(prefix.length), 'base64');
|
|
604
|
-
const pngSignature = Buffer.from([
|
|
605
|
-
137,
|
|
606
|
-
80,
|
|
607
|
-
78,
|
|
608
|
-
71,
|
|
609
|
-
13,
|
|
610
|
-
10,
|
|
611
|
-
26,
|
|
612
|
-
10
|
|
613
|
-
]);
|
|
614
|
-
if (buffer.length < 24 || !buffer.subarray(0, pngSignature.length).equals(pngSignature)) throw new Error('Screenshot returned invalid PNG data');
|
|
615
|
-
const width = buffer.readUInt32BE(16);
|
|
616
|
-
const height = buffer.readUInt32BE(20);
|
|
617
|
-
if (width <= 0 || height <= 0) throw new Error('Screenshot returned invalid PNG dimensions');
|
|
618
|
-
return {
|
|
619
|
-
width,
|
|
620
|
-
height
|
|
621
|
-
};
|
|
622
|
-
}
|
|
623
|
-
function assertLegacyWindowsCoordinateCompatibility({ geometry, screenshotBase64, inputSize }) {
|
|
624
|
-
if (!geometry?.primary) throw new Error('Windows legacy coordinate compatibility mode only supports the primary display');
|
|
625
|
-
if (0 !== geometry.bounds.x || 0 !== geometry.bounds.y) throw new Error(`Windows legacy coordinate compatibility mode requires a primary display origin of (0, 0), got (${geometry.bounds.x}, ${geometry.bounds.y})`);
|
|
626
|
-
const screenshotSize = readPngDimensions(screenshotBase64);
|
|
627
|
-
const displaySize = {
|
|
628
|
-
width: Math.round(geometry.bounds.width),
|
|
629
|
-
height: Math.round(geometry.bounds.height)
|
|
630
|
-
};
|
|
631
|
-
const sizesMatch = screenshotSize.width === displaySize.width && screenshotSize.height === displaySize.height && inputSize.width === displaySize.width && inputSize.height === displaySize.height;
|
|
632
|
-
if (!sizesMatch) throw new Error(`Windows legacy coordinate compatibility check failed: display=${displaySize.width}x${displaySize.height}, screenshot=${screenshotSize.width}x${screenshotSize.height}, input=${inputSize.width}x${inputSize.height}. Refusing to continue because clicks may use incorrect coordinates.`);
|
|
633
|
-
}
|
|
634
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
635
|
-
if (!displays.length) return;
|
|
636
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
637
|
-
return displays.find((display)=>display.id === displayId);
|
|
638
|
-
}
|
|
639
533
|
function windows_pointer_define_property(obj, key, value) {
|
|
640
534
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
641
535
|
value: value,
|
|
@@ -954,6 +848,28 @@ const EDGE_SCROLL_SPEC = {
|
|
|
954
848
|
function escapePowershellSingleQuoted(value) {
|
|
955
849
|
return value.replace(/'/g, "''");
|
|
956
850
|
}
|
|
851
|
+
function readWindowsDisplayGeometries() {
|
|
852
|
+
const script = `
|
|
853
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
854
|
+
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
855
|
+
$b = $_.Bounds
|
|
856
|
+
[PSCustomObject]@{
|
|
857
|
+
id = $_.DeviceName
|
|
858
|
+
name = $_.DeviceName
|
|
859
|
+
primary = $_.Primary
|
|
860
|
+
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
ConvertTo-Json @($s) -Compress
|
|
864
|
+
`.trim();
|
|
865
|
+
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
866
|
+
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
867
|
+
const parsed = JSON.parse(output);
|
|
868
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
869
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
870
|
+
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
871
|
+
return displays;
|
|
872
|
+
}
|
|
957
873
|
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
958
874
|
return geometries.map((display)=>({
|
|
959
875
|
id: display.id,
|
|
@@ -1057,13 +973,23 @@ function getDisplayInfoBinary() {
|
|
|
1057
973
|
displayInfoBinaryPath = binPath;
|
|
1058
974
|
return binPath;
|
|
1059
975
|
}
|
|
1060
|
-
function
|
|
976
|
+
function isFiniteNumber(value) {
|
|
1061
977
|
return 'number' == typeof value && Number.isFinite(value);
|
|
1062
978
|
}
|
|
979
|
+
function isDisplayBounds(value) {
|
|
980
|
+
if (!value || 'object' != typeof value) return false;
|
|
981
|
+
const bounds = value;
|
|
982
|
+
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
983
|
+
}
|
|
984
|
+
function isWindowsDisplayGeometry(value) {
|
|
985
|
+
if (!value || 'object' != typeof value) return false;
|
|
986
|
+
const candidate = value;
|
|
987
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
988
|
+
}
|
|
1063
989
|
function isDarwinDisplayGeometry(value) {
|
|
1064
990
|
if (!value || 'object' != typeof value) return false;
|
|
1065
991
|
const candidate = value;
|
|
1066
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds &&
|
|
992
|
+
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && isFiniteNumber(candidate.bounds.x) && isFiniteNumber(candidate.bounds.y) && isFiniteNumber(candidate.bounds.width) && isFiniteNumber(candidate.bounds.height);
|
|
1067
993
|
}
|
|
1068
994
|
function readDarwinDisplayGeometries() {
|
|
1069
995
|
const bin = getDisplayInfoBinary();
|
|
@@ -1133,6 +1059,11 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
1133
1059
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1134
1060
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1135
1061
|
}
|
|
1062
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
1063
|
+
if (!displays.length) return;
|
|
1064
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
1065
|
+
return displays.find((display)=>display.id === displayId);
|
|
1066
|
+
}
|
|
1136
1067
|
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1137
1068
|
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1138
1069
|
if ('win32' === process.platform) {
|
|
@@ -1225,24 +1156,13 @@ function normalizePrimaryKey(key) {
|
|
|
1225
1156
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1226
1157
|
}
|
|
1227
1158
|
class ComputerDevice {
|
|
1228
|
-
usesPhysicalWindowsCoordinates() {
|
|
1229
|
-
return 'win32' === process.platform && 'physical' === this.windowsCoordinateMode;
|
|
1230
|
-
}
|
|
1231
|
-
assertLegacyWindowsCoordinateCompatibility(screenshotBase64) {
|
|
1232
|
-
if ('win32' !== process.platform || 'legacy' !== this.windowsCoordinateMode) return;
|
|
1233
|
-
assertLegacyWindowsCoordinateCompatibility({
|
|
1234
|
-
geometry: this.displayGeometry,
|
|
1235
|
-
screenshotBase64,
|
|
1236
|
-
inputSize: this.inputDriver.getScreenSize()
|
|
1237
|
-
});
|
|
1238
|
-
}
|
|
1239
1159
|
async moveGlobalPointer(point, context, smooth) {
|
|
1240
1160
|
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1241
1161
|
const target = {
|
|
1242
1162
|
x: Math.round(point.x),
|
|
1243
1163
|
y: Math.round(point.y)
|
|
1244
1164
|
};
|
|
1245
|
-
if (
|
|
1165
|
+
if ('win32' === process.platform) {
|
|
1246
1166
|
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1247
1167
|
smoothSteps: smooth?.smoothSteps,
|
|
1248
1168
|
smoothDelayMs: smooth?.smoothDelay
|
|
@@ -1294,7 +1214,7 @@ class ComputerDevice {
|
|
|
1294
1214
|
}
|
|
1295
1215
|
static async listDisplays() {
|
|
1296
1216
|
try {
|
|
1297
|
-
if ('win32' === process.platform) return listWindowsDisplays(
|
|
1217
|
+
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1298
1218
|
const displays = await external_screenshot_desktop_default().listDisplays();
|
|
1299
1219
|
return displays.map((d)=>({
|
|
1300
1220
|
id: String(d.id),
|
|
@@ -1332,10 +1252,7 @@ class ComputerDevice {
|
|
|
1332
1252
|
}
|
|
1333
1253
|
}
|
|
1334
1254
|
device_libnut = await getLibnut();
|
|
1335
|
-
const
|
|
1336
|
-
const windowsDisplayGeometries = windowsDisplayDiscovery?.geometries;
|
|
1337
|
-
if (windowsDisplayDiscovery?.coordinateMode === 'legacy') warnDevice('Windows physical-pixel display enumeration returned no data; using the legacy Windows coordinate path for this device connection.');
|
|
1338
|
-
this.windowsCoordinateMode = windowsDisplayDiscovery?.coordinateMode ?? 'physical';
|
|
1255
|
+
const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
|
|
1339
1256
|
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1340
1257
|
const size = await this.size();
|
|
1341
1258
|
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
@@ -1346,7 +1263,6 @@ Platform: ${process.platform}
|
|
|
1346
1263
|
Display: ${this.displayId || 'Primary'}
|
|
1347
1264
|
Screen Size: ${size.width}x${size.height}
|
|
1348
1265
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1349
|
-
${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordinateMode}` : ''}
|
|
1350
1266
|
`;
|
|
1351
1267
|
debugDevice('Computer device connected', this.description);
|
|
1352
1268
|
await this.healthCheck(displays);
|
|
@@ -1370,7 +1286,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1370
1286
|
}
|
|
1371
1287
|
async healthCheck(displays) {
|
|
1372
1288
|
console.log('[HealthCheck] Starting health check...');
|
|
1373
|
-
console.log("[HealthCheck] @midscene/computer v1.13.1
|
|
1289
|
+
console.log("[HealthCheck] @midscene/computer v1.13.1");
|
|
1374
1290
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1375
1291
|
const screenshotTimeout = 15000;
|
|
1376
1292
|
let timeoutId;
|
|
@@ -1382,12 +1298,10 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1382
1298
|
timeoutPromise
|
|
1383
1299
|
]);
|
|
1384
1300
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1385
|
-
this.assertLegacyWindowsCoordinateCompatibility(base64);
|
|
1386
1301
|
console.log('[HealthCheck] Verifying mouse control...');
|
|
1387
|
-
const
|
|
1388
|
-
const startPos = usePhysicalWindowsCoordinates ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1302
|
+
const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1389
1303
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1390
|
-
if (
|
|
1304
|
+
if ('win32' === process.platform) {
|
|
1391
1305
|
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1392
1306
|
const bounds = this.displayGeometry.bounds;
|
|
1393
1307
|
const target = {
|
|
@@ -1453,7 +1367,7 @@ ${'win32' === process.platform ? `Windows Coordinate Mode: ${this.windowsCoordin
|
|
|
1453
1367
|
debugDevice('Taking screenshot', {
|
|
1454
1368
|
displayId: this.displayId
|
|
1455
1369
|
});
|
|
1456
|
-
if (
|
|
1370
|
+
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
1457
1371
|
const options = {
|
|
1458
1372
|
format: 'png'
|
|
1459
1373
|
};
|
|
@@ -1616,7 +1530,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1616
1530
|
this.inputDriver.sendKey(key, modifiers);
|
|
1617
1531
|
}
|
|
1618
1532
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1619
|
-
if (
|
|
1533
|
+
if ('win32' === process.platform) {
|
|
1620
1534
|
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1621
1535
|
if (activeWindowRect) {
|
|
1622
1536
|
const activeWindowCenter = {
|
|
@@ -1760,7 +1674,6 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1760
1674
|
device_define_property(this, "options", void 0);
|
|
1761
1675
|
device_define_property(this, "displayId", void 0);
|
|
1762
1676
|
device_define_property(this, "displayGeometry", void 0);
|
|
1763
|
-
device_define_property(this, "windowsCoordinateMode", 'physical');
|
|
1764
1677
|
device_define_property(this, "description", void 0);
|
|
1765
1678
|
device_define_property(this, "destroyed", false);
|
|
1766
1679
|
device_define_property(this, "xvfbInstance", void 0);
|
|
@@ -2907,7 +2820,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2907
2820
|
}
|
|
2908
2821
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2909
2822
|
function version() {
|
|
2910
|
-
const currentVersion = "1.13.1
|
|
2823
|
+
const currentVersion = "1.13.1";
|
|
2911
2824
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2912
2825
|
return currentVersion;
|
|
2913
2826
|
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -70,7 +70,6 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
70
70
|
private options?;
|
|
71
71
|
private displayId?;
|
|
72
72
|
private displayGeometry?;
|
|
73
|
-
private windowsCoordinateMode;
|
|
74
73
|
private description?;
|
|
75
74
|
private destroyed;
|
|
76
75
|
private xvfbInstance?;
|
|
@@ -88,8 +87,6 @@ export declare class ComputerDevice implements AbstractInterface {
|
|
|
88
87
|
uri?: string;
|
|
89
88
|
readonly inputPrimitives: ComputerInputPrimitives;
|
|
90
89
|
constructor(options?: ComputerDeviceOpt);
|
|
91
|
-
private usesPhysicalWindowsCoordinates;
|
|
92
|
-
private assertLegacyWindowsCoordinateCompatibility;
|
|
93
90
|
private moveGlobalPointer;
|
|
94
91
|
private moveDisplayPointer;
|
|
95
92
|
private performPointerDrag;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/computer",
|
|
3
|
-
"version": "1.13.1
|
|
3
|
+
"version": "1.13.1",
|
|
4
4
|
"description": "Midscene.js Computer Desktop Automation",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -33,15 +33,15 @@
|
|
|
33
33
|
"@computer-use/libnut": "^4.2.0",
|
|
34
34
|
"clipboardy": "^4.0.0",
|
|
35
35
|
"screenshot-desktop": "^1.15.3",
|
|
36
|
-
"@midscene/
|
|
37
|
-
"@midscene/
|
|
36
|
+
"@midscene/shared": "1.13.1",
|
|
37
|
+
"@midscene/core": "1.13.1"
|
|
38
38
|
},
|
|
39
39
|
"optionalDependencies": {
|
|
40
40
|
"node-mac-permissions": "2.5.0"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@rslib/core": "^0.18.3",
|
|
44
|
-
"@rstest/core": "0.
|
|
44
|
+
"@rstest/core": "0.12.0",
|
|
45
45
|
"@types/node": "^18.0.0",
|
|
46
46
|
"@types/screenshot-desktop": "1.15.0",
|
|
47
47
|
"dotenv": "^16.4.5",
|