@midscene/computer 1.12.3-beta-20260902081623.0 → 1.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +365 -109
- package/dist/es/index.mjs +365 -109
- package/dist/lib/cli.js +365 -109
- package/dist/lib/index.js +365 -109
- package/dist/types/index.d.ts +7 -9
- package/package.json +3 -3
package/dist/es/cli.mjs
CHANGED
|
@@ -29,6 +29,7 @@ function _define_property(obj, key, value) {
|
|
|
29
29
|
else obj[key] = value;
|
|
30
30
|
return obj;
|
|
31
31
|
}
|
|
32
|
+
const MOUSE_COORDINATE_TOLERANCE_PX = 5;
|
|
32
33
|
class ComputerInputDriver {
|
|
33
34
|
destroy() {
|
|
34
35
|
if (this.destroyed) return;
|
|
@@ -44,6 +45,15 @@ class ComputerInputDriver {
|
|
|
44
45
|
moveMouse(x, y) {
|
|
45
46
|
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
46
47
|
}
|
|
48
|
+
assertMousePosition(targetX, targetY, context) {
|
|
49
|
+
const current = this.getMousePos();
|
|
50
|
+
const drift = {
|
|
51
|
+
x: current.x - targetX,
|
|
52
|
+
y: current.y - targetY
|
|
53
|
+
};
|
|
54
|
+
if (Math.abs(drift.x) > MOUSE_COORDINATE_TOLERANCE_PX || Math.abs(drift.y) > MOUSE_COORDINATE_TOLERANCE_PX) throw new Error(`${context}: expected (${targetX}, ${targetY}), got (${current.x}, ${current.y}), drift=(${drift.x}, ${drift.y})`);
|
|
55
|
+
return drift;
|
|
56
|
+
}
|
|
47
57
|
focusActiveWindow() {
|
|
48
58
|
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
49
59
|
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
@@ -57,20 +67,6 @@ class ComputerInputDriver {
|
|
|
57
67
|
return false;
|
|
58
68
|
}
|
|
59
69
|
}
|
|
60
|
-
getActiveWindowRect() {
|
|
61
|
-
const lib = this.getLibnutOrThrow('getActiveWindowRect');
|
|
62
|
-
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
|
|
63
|
-
try {
|
|
64
|
-
const handle = lib.getActiveWindow();
|
|
65
|
-
if (!handle) return null;
|
|
66
|
-
const rect = lib.getWindowRect(handle);
|
|
67
|
-
if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
|
|
68
|
-
return rect;
|
|
69
|
-
} catch (error) {
|
|
70
|
-
this.options.debug(`getActiveWindowRect failed: ${error}`);
|
|
71
|
-
return null;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
70
|
mouseClick(button, double) {
|
|
75
71
|
const lib = this.getLibnutOrThrow('mouseClick');
|
|
76
72
|
if (void 0 !== double) lib.mouseClick(button, double);
|
|
@@ -184,6 +180,207 @@ class ComputerInputDriver {
|
|
|
184
180
|
this.pendingInputDelayWaits = new Set();
|
|
185
181
|
}
|
|
186
182
|
}
|
|
183
|
+
const POWERSHELL_TIMEOUT_MS = 15000;
|
|
184
|
+
const POWERSHELL_MAX_BUFFER = 67108864;
|
|
185
|
+
const WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE = `
|
|
186
|
+
$midsceneAssemblyName = New-Object System.Reflection.AssemblyName('MidsceneDpiNative')
|
|
187
|
+
$midsceneAssembly = [System.AppDomain]::CurrentDomain.DefineDynamicAssembly(
|
|
188
|
+
$midsceneAssemblyName,
|
|
189
|
+
[System.Reflection.Emit.AssemblyBuilderAccess]::Run
|
|
190
|
+
)
|
|
191
|
+
$midsceneModule = $midsceneAssembly.DefineDynamicModule('MidsceneDpiNativeModule')
|
|
192
|
+
$midsceneType = $midsceneModule.DefineType(
|
|
193
|
+
'MidsceneDpiNative.User32',
|
|
194
|
+
[System.Reflection.TypeAttributes]'Public, Class, Sealed, Abstract'
|
|
195
|
+
)
|
|
196
|
+
$midsceneMethodAttributes =
|
|
197
|
+
[System.Reflection.MethodAttributes]::Public -bor
|
|
198
|
+
[System.Reflection.MethodAttributes]::Static -bor
|
|
199
|
+
[System.Reflection.MethodAttributes]::PinvokeImpl
|
|
200
|
+
$midsceneSetDpiMethod = $midsceneType.DefinePInvokeMethod(
|
|
201
|
+
'SetThreadDpiAwarenessContext',
|
|
202
|
+
'user32.dll',
|
|
203
|
+
$midsceneMethodAttributes,
|
|
204
|
+
[System.Reflection.CallingConventions]::Standard,
|
|
205
|
+
[System.IntPtr],
|
|
206
|
+
[System.Type[]]@([System.IntPtr]),
|
|
207
|
+
[System.Runtime.InteropServices.CallingConvention]::Winapi,
|
|
208
|
+
[System.Runtime.InteropServices.CharSet]::None
|
|
209
|
+
)
|
|
210
|
+
$midsceneSetDpiMethod.SetImplementationFlags(
|
|
211
|
+
$midsceneSetDpiMethod.GetMethodImplementationFlags() -bor
|
|
212
|
+
[System.Reflection.MethodImplAttributes]::PreserveSig
|
|
213
|
+
)
|
|
214
|
+
$midsceneGetForegroundWindowMethod = $midsceneType.DefinePInvokeMethod(
|
|
215
|
+
'GetForegroundWindow',
|
|
216
|
+
'user32.dll',
|
|
217
|
+
$midsceneMethodAttributes,
|
|
218
|
+
[System.Reflection.CallingConventions]::Standard,
|
|
219
|
+
[System.IntPtr],
|
|
220
|
+
[System.Type[]]@(),
|
|
221
|
+
[System.Runtime.InteropServices.CallingConvention]::Winapi,
|
|
222
|
+
[System.Runtime.InteropServices.CharSet]::None
|
|
223
|
+
)
|
|
224
|
+
$midsceneGetForegroundWindowMethod.SetImplementationFlags(
|
|
225
|
+
$midsceneGetForegroundWindowMethod.GetMethodImplementationFlags() -bor
|
|
226
|
+
[System.Reflection.MethodImplAttributes]::PreserveSig
|
|
227
|
+
)
|
|
228
|
+
$midsceneGetWindowRectMethod = $midsceneType.DefinePInvokeMethod(
|
|
229
|
+
'GetWindowRect',
|
|
230
|
+
'user32.dll',
|
|
231
|
+
$midsceneMethodAttributes,
|
|
232
|
+
[System.Reflection.CallingConventions]::Standard,
|
|
233
|
+
[bool],
|
|
234
|
+
[System.Type[]]@([System.IntPtr], [System.IntPtr]),
|
|
235
|
+
[System.Runtime.InteropServices.CallingConvention]::Winapi,
|
|
236
|
+
[System.Runtime.InteropServices.CharSet]::None
|
|
237
|
+
)
|
|
238
|
+
$midsceneGetWindowRectMethod.SetImplementationFlags(
|
|
239
|
+
$midsceneGetWindowRectMethod.GetMethodImplementationFlags() -bor
|
|
240
|
+
[System.Reflection.MethodImplAttributes]::PreserveSig
|
|
241
|
+
)
|
|
242
|
+
$midsceneNativeMethods = $midsceneType.CreateType()
|
|
243
|
+
$midscenePerMonitorV2 = [System.IntPtr](-4)
|
|
244
|
+
$midscenePreviousDpiContext =
|
|
245
|
+
$midsceneNativeMethods::SetThreadDpiAwarenessContext($midscenePerMonitorV2)
|
|
246
|
+
if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
247
|
+
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
248
|
+
}
|
|
249
|
+
`.trim();
|
|
250
|
+
function runWindowsPhysicalPixelPowershell(script) {
|
|
251
|
+
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
252
|
+
$ErrorActionPreference = 'Stop'
|
|
253
|
+
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
254
|
+
${script}`;
|
|
255
|
+
const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
|
|
256
|
+
return execFileSync('powershell.exe', [
|
|
257
|
+
'-NoProfile',
|
|
258
|
+
'-NonInteractive',
|
|
259
|
+
'-EncodedCommand',
|
|
260
|
+
encoded
|
|
261
|
+
], {
|
|
262
|
+
encoding: 'utf8',
|
|
263
|
+
timeout: POWERSHELL_TIMEOUT_MS,
|
|
264
|
+
maxBuffer: POWERSHELL_MAX_BUFFER,
|
|
265
|
+
windowsHide: true
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
function windows_pointer_define_property(obj, key, value) {
|
|
269
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
270
|
+
value: value,
|
|
271
|
+
enumerable: true,
|
|
272
|
+
configurable: true,
|
|
273
|
+
writable: true
|
|
274
|
+
});
|
|
275
|
+
else obj[key] = value;
|
|
276
|
+
return obj;
|
|
277
|
+
}
|
|
278
|
+
const WINDOWS_POINTER_TOLERANCE_PX = 5;
|
|
279
|
+
function assertFinitePoint(point, context) {
|
|
280
|
+
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) throw new Error(`${context} must contain finite coordinates, got (${point.x}, ${point.y})`);
|
|
281
|
+
}
|
|
282
|
+
function parseWindowsPointerPosition(output, context) {
|
|
283
|
+
const match = output.trim().match(/^(-?\d+),(-?\d+)$/);
|
|
284
|
+
if (!match) throw new Error(`${context} returned an invalid cursor position: ${JSON.stringify(output.trim())}`);
|
|
285
|
+
return {
|
|
286
|
+
x: Number(match[1]),
|
|
287
|
+
y: Number(match[2])
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
function windowsPointerPositionScript() {
|
|
291
|
+
return `
|
|
292
|
+
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
293
|
+
$position = [System.Windows.Forms.Cursor]::Position
|
|
294
|
+
[Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
|
|
295
|
+
`.trim();
|
|
296
|
+
}
|
|
297
|
+
function windowsPointerMoveScript(point, options) {
|
|
298
|
+
assertFinitePoint(point, 'Windows pointer target');
|
|
299
|
+
if (options?.smoothSteps !== void 0 && !Number.isFinite(options.smoothSteps) || options?.smoothDelayMs !== void 0 && !Number.isFinite(options.smoothDelayMs)) throw new Error('Windows pointer smoothing options must be finite numbers');
|
|
300
|
+
const targetX = Math.round(point.x);
|
|
301
|
+
const targetY = Math.round(point.y);
|
|
302
|
+
const smoothSteps = Math.max(1, Math.round(options?.smoothSteps ?? 1));
|
|
303
|
+
const smoothDelayMs = Math.max(0, Math.round(options?.smoothDelayMs ?? 0));
|
|
304
|
+
return `
|
|
305
|
+
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
306
|
+
$targetX = ${targetX}
|
|
307
|
+
$targetY = ${targetY}
|
|
308
|
+
$smoothSteps = ${smoothSteps}
|
|
309
|
+
$smoothDelayMs = ${smoothDelayMs}
|
|
310
|
+
$start = [System.Windows.Forms.Cursor]::Position
|
|
311
|
+
for ($step = 1; $step -le $smoothSteps; $step += 1) {
|
|
312
|
+
$x = [int][Math]::Round($start.X + (($targetX - $start.X) * $step / [double]$smoothSteps))
|
|
313
|
+
$y = [int][Math]::Round($start.Y + (($targetY - $start.Y) * $step / [double]$smoothSteps))
|
|
314
|
+
[System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($x, $y)
|
|
315
|
+
if ($smoothDelayMs -gt 0 -and $step -lt $smoothSteps) {
|
|
316
|
+
Start-Sleep -Milliseconds $smoothDelayMs
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
[System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($targetX, $targetY)
|
|
320
|
+
$position = [System.Windows.Forms.Cursor]::Position
|
|
321
|
+
[Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
|
|
322
|
+
`.trim();
|
|
323
|
+
}
|
|
324
|
+
function parseWindowsWindowRect(output) {
|
|
325
|
+
const value = output.trim();
|
|
326
|
+
if (!value) return null;
|
|
327
|
+
const match = value.match(/^(-?\d+),(-?\d+),(\d+),(\d+)$/);
|
|
328
|
+
if (!match) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
|
|
329
|
+
const rect = {
|
|
330
|
+
x: Number(match[1]),
|
|
331
|
+
y: Number(match[2]),
|
|
332
|
+
width: Number(match[3]),
|
|
333
|
+
height: Number(match[4])
|
|
334
|
+
};
|
|
335
|
+
if (rect.width <= 0 || rect.height <= 0) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
|
|
336
|
+
return rect;
|
|
337
|
+
}
|
|
338
|
+
function windowsActiveWindowRectScript() {
|
|
339
|
+
return `
|
|
340
|
+
$midsceneWindowHandle = $midsceneNativeMethods::GetForegroundWindow()
|
|
341
|
+
if ($midsceneWindowHandle -eq [System.IntPtr]::Zero) {
|
|
342
|
+
return
|
|
343
|
+
}
|
|
344
|
+
$midsceneWindowRectBuffer = [System.Runtime.InteropServices.Marshal]::AllocHGlobal(16)
|
|
345
|
+
try {
|
|
346
|
+
if (-not $midsceneNativeMethods::GetWindowRect($midsceneWindowHandle, $midsceneWindowRectBuffer)) {
|
|
347
|
+
throw 'GetWindowRect failed for the foreground window.'
|
|
348
|
+
}
|
|
349
|
+
$left = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 0)
|
|
350
|
+
$top = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 4)
|
|
351
|
+
$right = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 8)
|
|
352
|
+
$bottom = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 12)
|
|
353
|
+
[Console]::Out.Write(('{0},{1},{2},{3}' -f $left, $top, ($right - $left), ($bottom - $top)))
|
|
354
|
+
}
|
|
355
|
+
finally {
|
|
356
|
+
[System.Runtime.InteropServices.Marshal]::FreeHGlobal($midsceneWindowRectBuffer)
|
|
357
|
+
}
|
|
358
|
+
`.trim();
|
|
359
|
+
}
|
|
360
|
+
class WindowsPointerDriver {
|
|
361
|
+
getPosition() {
|
|
362
|
+
return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerPositionScript()), 'Windows pointer query');
|
|
363
|
+
}
|
|
364
|
+
getActiveWindowRect() {
|
|
365
|
+
return parseWindowsWindowRect(this.options.runPhysicalPixelPowershell(windowsActiveWindowRectScript()));
|
|
366
|
+
}
|
|
367
|
+
moveTo(point, options) {
|
|
368
|
+
return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerMoveScript(point, options)), 'Windows pointer move');
|
|
369
|
+
}
|
|
370
|
+
constructor(options){
|
|
371
|
+
windows_pointer_define_property(this, "options", void 0);
|
|
372
|
+
this.options = options;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
function windowsPointerDrift(expected, actual) {
|
|
376
|
+
return {
|
|
377
|
+
x: actual.x - expected.x,
|
|
378
|
+
y: actual.y - expected.y
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
function windowsPointerIsWithinTolerance(drift) {
|
|
382
|
+
return Math.abs(drift.x) <= WINDOWS_POINTER_TOLERANCE_PX && Math.abs(drift.y) <= WINDOWS_POINTER_TOLERANCE_PX;
|
|
383
|
+
}
|
|
187
384
|
const debugXvfb = getDebug('computer:xvfb');
|
|
188
385
|
const xvfbCleanupMonitorScript = String.raw`
|
|
189
386
|
const parentPid = Number(process.argv[1]);
|
|
@@ -441,39 +638,36 @@ function sendKeyViaAppleScript(key, modifiers = []) {
|
|
|
441
638
|
script
|
|
442
639
|
]);
|
|
443
640
|
}
|
|
444
|
-
const POWERSHELL_TIMEOUT_MS = 15000;
|
|
445
|
-
const POWERSHELL_MAX_BUFFER = 67108864;
|
|
446
641
|
function escapePowershellSingleQuoted(value) {
|
|
447
642
|
return value.replace(/'/g, "''");
|
|
448
643
|
}
|
|
449
|
-
function
|
|
450
|
-
const prefixed = `$ProgressPreference = 'SilentlyContinue'\n${script}`;
|
|
451
|
-
const encoded = Buffer.from(prefixed, 'utf16le').toString('base64');
|
|
452
|
-
return execFileSync('powershell.exe', [
|
|
453
|
-
'-NoProfile',
|
|
454
|
-
'-NonInteractive',
|
|
455
|
-
'-EncodedCommand',
|
|
456
|
-
encoded
|
|
457
|
-
], {
|
|
458
|
-
encoding: 'utf8',
|
|
459
|
-
timeout: POWERSHELL_TIMEOUT_MS,
|
|
460
|
-
maxBuffer: POWERSHELL_MAX_BUFFER,
|
|
461
|
-
windowsHide: true
|
|
462
|
-
});
|
|
463
|
-
}
|
|
464
|
-
function listWindowsDisplays() {
|
|
644
|
+
function readWindowsDisplayGeometries() {
|
|
465
645
|
const script = `
|
|
466
646
|
Add-Type -AssemblyName System.Windows.Forms
|
|
467
647
|
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
468
|
-
|
|
648
|
+
$b = $_.Bounds
|
|
649
|
+
[PSCustomObject]@{
|
|
650
|
+
id = $_.DeviceName
|
|
651
|
+
name = $_.DeviceName
|
|
652
|
+
primary = $_.Primary
|
|
653
|
+
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
654
|
+
}
|
|
469
655
|
}
|
|
470
656
|
ConvertTo-Json @($s) -Compress
|
|
471
657
|
`.trim();
|
|
472
|
-
const
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
658
|
+
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
659
|
+
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
660
|
+
const parsed = JSON.parse(output);
|
|
661
|
+
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
662
|
+
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
663
|
+
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
664
|
+
return displays;
|
|
665
|
+
}
|
|
666
|
+
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
667
|
+
return geometries.map((display)=>({
|
|
668
|
+
id: display.id,
|
|
669
|
+
name: display.name,
|
|
670
|
+
primary: display.primary
|
|
477
671
|
}));
|
|
478
672
|
}
|
|
479
673
|
let device_libnut = null;
|
|
@@ -575,6 +769,16 @@ function getDisplayInfoBinary() {
|
|
|
575
769
|
function isFiniteNumber(value) {
|
|
576
770
|
return 'number' == typeof value && Number.isFinite(value);
|
|
577
771
|
}
|
|
772
|
+
function isDisplayBounds(value) {
|
|
773
|
+
if (!value || 'object' != typeof value) return false;
|
|
774
|
+
const bounds = value;
|
|
775
|
+
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
776
|
+
}
|
|
777
|
+
function isWindowsDisplayGeometry(value) {
|
|
778
|
+
if (!value || 'object' != typeof value) return false;
|
|
779
|
+
const candidate = value;
|
|
780
|
+
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
781
|
+
}
|
|
578
782
|
function isDarwinDisplayGeometry(value) {
|
|
579
783
|
if (!value || 'object' != typeof value) return false;
|
|
580
784
|
const candidate = value;
|
|
@@ -617,9 +821,11 @@ function readDarwinFrontmostApplication() {
|
|
|
617
821
|
return;
|
|
618
822
|
}
|
|
619
823
|
}
|
|
620
|
-
async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
|
|
621
|
-
|
|
622
|
-
|
|
824
|
+
async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, current, holdDuration, reason) {
|
|
825
|
+
const drift = {
|
|
826
|
+
x: current.x - targetX,
|
|
827
|
+
y: current.y - targetY
|
|
828
|
+
};
|
|
623
829
|
debugComputerInput('tap mouse moved %o', {
|
|
624
830
|
reason,
|
|
625
831
|
target: {
|
|
@@ -627,10 +833,7 @@ async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDurati
|
|
|
627
833
|
y: targetY
|
|
628
834
|
},
|
|
629
835
|
current,
|
|
630
|
-
drift
|
|
631
|
-
x: current.x - targetX,
|
|
632
|
-
y: current.y - targetY
|
|
633
|
-
}
|
|
836
|
+
drift
|
|
634
837
|
});
|
|
635
838
|
await inputDriver.withMouseButton('left', async ()=>{
|
|
636
839
|
debugComputerInput('tap mouse down %o', {
|
|
@@ -649,9 +852,18 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
649
852
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
650
853
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
651
854
|
}
|
|
652
|
-
function
|
|
653
|
-
if (
|
|
654
|
-
|
|
855
|
+
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
856
|
+
if (!displays.length) return;
|
|
857
|
+
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
858
|
+
return displays.find((display)=>display.id === displayId);
|
|
859
|
+
}
|
|
860
|
+
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
861
|
+
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
862
|
+
if ('win32' === process.platform) {
|
|
863
|
+
const geometry = resolveWindowsDisplayGeometryFromList(displayId, windowsDisplays ?? readWindowsDisplayGeometries());
|
|
864
|
+
if (!geometry) throw new Error(displayId ? `Requested Windows display not found: ${displayId}` : 'No Windows displays were detected');
|
|
865
|
+
return geometry;
|
|
866
|
+
}
|
|
655
867
|
}
|
|
656
868
|
function mapDisplayLocalPointToGlobal(point, geometry) {
|
|
657
869
|
if (!geometry) return point;
|
|
@@ -737,6 +949,30 @@ function normalizePrimaryKey(key) {
|
|
|
737
949
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
738
950
|
}
|
|
739
951
|
class ComputerDevice {
|
|
952
|
+
async moveGlobalPointer(point, context, smooth) {
|
|
953
|
+
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
954
|
+
const target = {
|
|
955
|
+
x: Math.round(point.x),
|
|
956
|
+
y: Math.round(point.y)
|
|
957
|
+
};
|
|
958
|
+
if ('win32' === process.platform) {
|
|
959
|
+
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
960
|
+
smoothSteps: smooth?.smoothSteps,
|
|
961
|
+
smoothDelayMs: smooth?.smoothDelay
|
|
962
|
+
});
|
|
963
|
+
const drift = windowsPointerDrift(target, actual);
|
|
964
|
+
if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`${context}: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
|
|
965
|
+
await this.inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
966
|
+
return actual;
|
|
967
|
+
}
|
|
968
|
+
if (smooth) await this.inputDriver.smoothMoveMouse(target.x, target.y, smooth.smoothSteps, smooth.smoothDelay);
|
|
969
|
+
else this.inputDriver.moveMouse(target.x, target.y);
|
|
970
|
+
await this.inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
971
|
+
return this.inputDriver.getMousePos();
|
|
972
|
+
}
|
|
973
|
+
moveDisplayPointer(point, context, smooth) {
|
|
974
|
+
return this.moveGlobalPointer(this.toGlobalPoint(point), context, smooth);
|
|
975
|
+
}
|
|
740
976
|
async focusKeyboardTarget(element, delayMs) {
|
|
741
977
|
const [x, y] = element.center;
|
|
742
978
|
if ('darwin' === process.platform) await this.inputPrimitives.pointer.tap({
|
|
@@ -744,11 +980,10 @@ class ComputerDevice {
|
|
|
744
980
|
y
|
|
745
981
|
});
|
|
746
982
|
else {
|
|
747
|
-
|
|
983
|
+
await this.moveDisplayPointer({
|
|
748
984
|
x,
|
|
749
985
|
y
|
|
750
|
-
});
|
|
751
|
-
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
986
|
+
}, 'Mouse did not reach the keyboard focus target');
|
|
752
987
|
this.inputDriver.mouseClick('left');
|
|
753
988
|
}
|
|
754
989
|
await this.inputDriver.delay(delayMs);
|
|
@@ -767,7 +1002,7 @@ class ComputerDevice {
|
|
|
767
1002
|
}));
|
|
768
1003
|
} catch (error) {
|
|
769
1004
|
debugDevice(`Failed to list displays: ${error}`);
|
|
770
|
-
|
|
1005
|
+
throw new Error(`Failed to list displays: ${error}`);
|
|
771
1006
|
}
|
|
772
1007
|
}
|
|
773
1008
|
async connect() {
|
|
@@ -796,9 +1031,10 @@ class ComputerDevice {
|
|
|
796
1031
|
}
|
|
797
1032
|
}
|
|
798
1033
|
device_libnut = await getLibnut();
|
|
799
|
-
|
|
1034
|
+
const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
|
|
1035
|
+
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
800
1036
|
const size = await this.size();
|
|
801
|
-
const displays = await ComputerDevice.listDisplays();
|
|
1037
|
+
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
802
1038
|
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
803
1039
|
this.description = `
|
|
804
1040
|
Type: Computer
|
|
@@ -808,7 +1044,7 @@ Screen Size: ${size.width}x${size.height}
|
|
|
808
1044
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
809
1045
|
`;
|
|
810
1046
|
debugDevice('Computer device connected', this.description);
|
|
811
|
-
await this.healthCheck();
|
|
1047
|
+
await this.healthCheck(displays);
|
|
812
1048
|
} catch (error) {
|
|
813
1049
|
if (this.xvfbInstance) {
|
|
814
1050
|
if (!this.options?.keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
@@ -827,9 +1063,9 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
827
1063
|
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
828
1064
|
}
|
|
829
1065
|
}
|
|
830
|
-
async healthCheck() {
|
|
1066
|
+
async healthCheck(displays) {
|
|
831
1067
|
console.log('[HealthCheck] Starting health check...');
|
|
832
|
-
console.log("[HealthCheck] @midscene/computer v1.12.3
|
|
1068
|
+
console.log("[HealthCheck] @midscene/computer v1.12.3");
|
|
833
1069
|
console.log('[HealthCheck] Taking screenshot...');
|
|
834
1070
|
const screenshotTimeout = 15000;
|
|
835
1071
|
let timeoutId;
|
|
@@ -841,23 +1077,38 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
841
1077
|
timeoutPromise
|
|
842
1078
|
]);
|
|
843
1079
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
844
|
-
console.log('[HealthCheck]
|
|
845
|
-
const startPos = this.inputDriver.getMousePos();
|
|
1080
|
+
console.log('[HealthCheck] Verifying mouse control...');
|
|
1081
|
+
const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
846
1082
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
1083
|
+
if ('win32' === process.platform) {
|
|
1084
|
+
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1085
|
+
const bounds = this.displayGeometry.bounds;
|
|
1086
|
+
const target = {
|
|
1087
|
+
x: Math.round(bounds.x + bounds.width / 2),
|
|
1088
|
+
y: Math.round(bounds.y + bounds.height / 2)
|
|
1089
|
+
};
|
|
1090
|
+
try {
|
|
1091
|
+
const actual = this.windowsPointerDriver.moveTo(target);
|
|
1092
|
+
const drift = windowsPointerDrift(target, actual);
|
|
1093
|
+
if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`Windows screenshot-space pointer verification: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
|
|
1094
|
+
} finally{
|
|
1095
|
+
this.windowsPointerDriver.moveTo(startPos);
|
|
1096
|
+
}
|
|
1097
|
+
console.log(`[HealthCheck] Mouse verified in screenshot-space display bounds (${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`);
|
|
1098
|
+
} else {
|
|
1099
|
+
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
1100
|
+
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
1101
|
+
const targetX = startPos.x + offsetX;
|
|
1102
|
+
const targetY = startPos.y + offsetY;
|
|
1103
|
+
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
1104
|
+
try {
|
|
1105
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
1106
|
+
await sleep(CLICK_SETTLE_DELAY);
|
|
1107
|
+
this.inputDriver.assertMousePosition(targetX, targetY, 'Mouse health check failed');
|
|
1108
|
+
} finally{
|
|
1109
|
+
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
1110
|
+
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
1111
|
+
}
|
|
861
1112
|
}
|
|
862
1113
|
if ('win32' === process.platform && !this.isRunningAsAdmin()) {
|
|
863
1114
|
const hint = [
|
|
@@ -867,10 +1118,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
867
1118
|
].join(' ');
|
|
868
1119
|
warnDevice(`[HealthCheck] ${hint}`);
|
|
869
1120
|
}
|
|
870
|
-
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
871
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
872
1121
|
console.log('[HealthCheck] Listing monitors...');
|
|
873
|
-
const displays = await ComputerDevice.listDisplays();
|
|
874
1122
|
if (displays.length > 0) {
|
|
875
1123
|
console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
|
|
876
1124
|
for (const display of displays){
|
|
@@ -938,7 +1186,6 @@ Original error: ${lastRawMessage}`);
|
|
|
938
1186
|
$screen = [System.Windows.Forms.Screen]::AllScreens | Where-Object { $_.DeviceName -eq $dn } | Select-Object -First 1
|
|
939
1187
|
if (-not $screen) { throw "Requested display not found: $dn" }` : '$screen = [System.Windows.Forms.Screen]::PrimaryScreen';
|
|
940
1188
|
const script = `
|
|
941
|
-
$ErrorActionPreference = 'Stop'
|
|
942
1189
|
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
943
1190
|
${selectScreen}
|
|
944
1191
|
$b = $screen.Bounds
|
|
@@ -952,7 +1199,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
952
1199
|
`.trim();
|
|
953
1200
|
let stdout;
|
|
954
1201
|
try {
|
|
955
|
-
stdout =
|
|
1202
|
+
stdout = runWindowsPhysicalPixelPowershell(script);
|
|
956
1203
|
} catch (error) {
|
|
957
1204
|
const message = error instanceof Error ? error.message : String(error);
|
|
958
1205
|
throw new Error(`Failed to take screenshot on Windows: ${message}`);
|
|
@@ -1054,11 +1301,15 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1054
1301
|
}
|
|
1055
1302
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1056
1303
|
if ('win32' === process.platform) {
|
|
1057
|
-
const activeWindowRect = this.
|
|
1058
|
-
if (activeWindowRect)
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1304
|
+
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1305
|
+
if (activeWindowRect) {
|
|
1306
|
+
const activeWindowCenter = {
|
|
1307
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
1308
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
1309
|
+
};
|
|
1310
|
+
const bounds = this.displayGeometry?.bounds;
|
|
1311
|
+
if (!bounds || activeWindowCenter.x >= bounds.x && activeWindowCenter.x < bounds.x + bounds.width && activeWindowCenter.y >= bounds.y && activeWindowCenter.y < bounds.y + bounds.height) return activeWindowCenter;
|
|
1312
|
+
}
|
|
1062
1313
|
}
|
|
1063
1314
|
return this.toGlobalPoint({
|
|
1064
1315
|
x: screenSize.width / 2,
|
|
@@ -1069,17 +1320,16 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1069
1320
|
if (param.locate) {
|
|
1070
1321
|
const element = param.locate;
|
|
1071
1322
|
const [x, y] = element.center;
|
|
1072
|
-
|
|
1323
|
+
await this.moveDisplayPointer({
|
|
1073
1324
|
x,
|
|
1074
1325
|
y
|
|
1075
|
-
});
|
|
1076
|
-
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1326
|
+
}, 'Mouse did not reach the scroll target');
|
|
1077
1327
|
return;
|
|
1078
1328
|
}
|
|
1079
1329
|
const screenSize = await this.size();
|
|
1080
1330
|
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
1081
1331
|
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
1082
|
-
this.
|
|
1332
|
+
await this.moveGlobalPointer(point, 'Mouse did not reach the scroll viewport');
|
|
1083
1333
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1084
1334
|
return screenSize;
|
|
1085
1335
|
}
|
|
@@ -1204,6 +1454,9 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1204
1454
|
runPhasedScroll,
|
|
1205
1455
|
debug: (message)=>debugDevice(message)
|
|
1206
1456
|
}));
|
|
1457
|
+
device_define_property(this, "windowsPointerDriver", new WindowsPointerDriver({
|
|
1458
|
+
runPhysicalPixelPowershell: runWindowsPhysicalPixelPowershell
|
|
1459
|
+
}));
|
|
1207
1460
|
device_define_property(this, "useAppleScript", void 0);
|
|
1208
1461
|
device_define_property(this, "adminCheckCache", void 0);
|
|
1209
1462
|
device_define_property(this, "uri", void 0);
|
|
@@ -1228,15 +1481,17 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1228
1481
|
},
|
|
1229
1482
|
holdDuration,
|
|
1230
1483
|
displayId: this.displayId,
|
|
1231
|
-
displayGeometry: this.displayGeometry
|
|
1232
|
-
screenIndex: this.displayGeometry.screenIndex,
|
|
1233
|
-
cgDisplayId: this.displayGeometry.cgDisplayId,
|
|
1234
|
-
bounds: this.displayGeometry.bounds
|
|
1235
|
-
} : void 0
|
|
1484
|
+
displayGeometry: this.displayGeometry
|
|
1236
1485
|
});
|
|
1237
1486
|
const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
|
|
1238
|
-
await this.
|
|
1239
|
-
|
|
1487
|
+
const current = await this.moveGlobalPointer({
|
|
1488
|
+
x: targetX,
|
|
1489
|
+
y: targetY
|
|
1490
|
+
}, 'Mouse did not reach the tap target', {
|
|
1491
|
+
smoothSteps: SMOOTH_MOVE_STEPS_TAP,
|
|
1492
|
+
smoothDelay: SMOOTH_MOVE_DELAY_TAP
|
|
1493
|
+
});
|
|
1494
|
+
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, current, holdDuration, 'primary');
|
|
1240
1495
|
if (frontmostBefore && 'darwin' === process.platform) {
|
|
1241
1496
|
await sleep(CLICK_FOCUS_SETTLE_DELAY);
|
|
1242
1497
|
const frontmostAfter = readDarwinFrontmostApplication();
|
|
@@ -1247,42 +1502,43 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1247
1502
|
focusChanged
|
|
1248
1503
|
});
|
|
1249
1504
|
if (focusChanged) {
|
|
1250
|
-
this.
|
|
1251
|
-
|
|
1505
|
+
const followUpCurrent = await this.moveGlobalPointer({
|
|
1506
|
+
x: targetX,
|
|
1507
|
+
y: targetY
|
|
1508
|
+
}, 'Mouse did not reach the focus follow-up target');
|
|
1509
|
+
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, followUpCurrent, holdDuration, 'focus-follow-up');
|
|
1252
1510
|
}
|
|
1253
1511
|
}
|
|
1254
1512
|
},
|
|
1255
1513
|
doubleClick: async ({ x, y })=>{
|
|
1256
|
-
|
|
1514
|
+
await this.moveDisplayPointer({
|
|
1257
1515
|
x,
|
|
1258
1516
|
y
|
|
1259
|
-
});
|
|
1260
|
-
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1517
|
+
}, 'Mouse did not reach the double-click target');
|
|
1261
1518
|
this.inputDriver.mouseClick('left', true);
|
|
1262
1519
|
},
|
|
1263
1520
|
rightClick: async ({ x, y })=>{
|
|
1264
|
-
|
|
1521
|
+
await this.moveDisplayPointer({
|
|
1265
1522
|
x,
|
|
1266
1523
|
y
|
|
1267
|
-
});
|
|
1268
|
-
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1524
|
+
}, 'Mouse did not reach the right-click target');
|
|
1269
1525
|
this.inputDriver.mouseClick('right');
|
|
1270
1526
|
},
|
|
1271
1527
|
hover: async ({ x, y })=>{
|
|
1272
|
-
|
|
1528
|
+
await this.moveDisplayPointer({
|
|
1273
1529
|
x,
|
|
1274
1530
|
y
|
|
1531
|
+
}, 'Mouse did not reach the hover target', {
|
|
1532
|
+
smoothSteps: SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1533
|
+
smoothDelay: SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
1275
1534
|
});
|
|
1276
|
-
await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
1277
1535
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1278
1536
|
},
|
|
1279
1537
|
dragAndDrop: async (from, to)=>{
|
|
1280
|
-
|
|
1281
|
-
const globalTo = this.toGlobalPoint(to);
|
|
1282
|
-
this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
|
|
1538
|
+
await this.moveDisplayPointer(from, 'Mouse did not reach the drag start target');
|
|
1283
1539
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1284
1540
|
await this.inputDriver.delay(100);
|
|
1285
|
-
this.
|
|
1541
|
+
await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target');
|
|
1286
1542
|
await this.inputDriver.delay(100);
|
|
1287
1543
|
});
|
|
1288
1544
|
}
|
|
@@ -2255,7 +2511,7 @@ const tools = new ComputerMidsceneTools({
|
|
|
2255
2511
|
});
|
|
2256
2512
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2257
2513
|
stripPrefix: 'computer_',
|
|
2258
|
-
version: "1.12.3
|
|
2514
|
+
version: "1.12.3",
|
|
2259
2515
|
extraCommands: createReportCliCommands()
|
|
2260
2516
|
}).catch((e)=>{
|
|
2261
2517
|
process.exit(reportCLIError(e));
|