@midscene/computer 1.12.3-beta-20260902081623.0 → 1.12.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/es/cli.mjs CHANGED
@@ -29,6 +29,7 @@ function _define_property(obj, key, value) {
29
29
  else obj[key] = value;
30
30
  return obj;
31
31
  }
32
+ const MOUSE_COORDINATE_TOLERANCE_PX = 5;
32
33
  class ComputerInputDriver {
33
34
  destroy() {
34
35
  if (this.destroyed) return;
@@ -44,6 +45,15 @@ class ComputerInputDriver {
44
45
  moveMouse(x, y) {
45
46
  this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
46
47
  }
48
+ assertMousePosition(targetX, targetY, context) {
49
+ const current = this.getMousePos();
50
+ const drift = {
51
+ x: current.x - targetX,
52
+ y: current.y - targetY
53
+ };
54
+ if (Math.abs(drift.x) > MOUSE_COORDINATE_TOLERANCE_PX || Math.abs(drift.y) > MOUSE_COORDINATE_TOLERANCE_PX) throw new Error(`${context}: expected (${targetX}, ${targetY}), got (${current.x}, ${current.y}), drift=(${drift.x}, ${drift.y})`);
55
+ return drift;
56
+ }
47
57
  focusActiveWindow() {
48
58
  const lib = this.getLibnutOrThrow('focusActiveWindow');
49
59
  if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
@@ -57,20 +67,6 @@ class ComputerInputDriver {
57
67
  return false;
58
68
  }
59
69
  }
60
- getActiveWindowRect() {
61
- const lib = this.getLibnutOrThrow('getActiveWindowRect');
62
- if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.getWindowRect) return null;
63
- try {
64
- const handle = lib.getActiveWindow();
65
- if (!handle) return null;
66
- const rect = lib.getWindowRect(handle);
67
- if (!Number.isFinite(rect.x) || !Number.isFinite(rect.y) || !Number.isFinite(rect.width) || !Number.isFinite(rect.height) || rect.width <= 0 || rect.height <= 0) return null;
68
- return rect;
69
- } catch (error) {
70
- this.options.debug(`getActiveWindowRect failed: ${error}`);
71
- return null;
72
- }
73
- }
74
70
  mouseClick(button, double) {
75
71
  const lib = this.getLibnutOrThrow('mouseClick');
76
72
  if (void 0 !== double) lib.mouseClick(button, double);
@@ -184,6 +180,207 @@ class ComputerInputDriver {
184
180
  this.pendingInputDelayWaits = new Set();
185
181
  }
186
182
  }
183
+ const POWERSHELL_TIMEOUT_MS = 15000;
184
+ const POWERSHELL_MAX_BUFFER = 67108864;
185
+ const WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE = `
186
+ $midsceneAssemblyName = New-Object System.Reflection.AssemblyName('MidsceneDpiNative')
187
+ $midsceneAssembly = [System.AppDomain]::CurrentDomain.DefineDynamicAssembly(
188
+ $midsceneAssemblyName,
189
+ [System.Reflection.Emit.AssemblyBuilderAccess]::Run
190
+ )
191
+ $midsceneModule = $midsceneAssembly.DefineDynamicModule('MidsceneDpiNativeModule')
192
+ $midsceneType = $midsceneModule.DefineType(
193
+ 'MidsceneDpiNative.User32',
194
+ [System.Reflection.TypeAttributes]'Public, Class, Sealed, Abstract'
195
+ )
196
+ $midsceneMethodAttributes =
197
+ [System.Reflection.MethodAttributes]::Public -bor
198
+ [System.Reflection.MethodAttributes]::Static -bor
199
+ [System.Reflection.MethodAttributes]::PinvokeImpl
200
+ $midsceneSetDpiMethod = $midsceneType.DefinePInvokeMethod(
201
+ 'SetThreadDpiAwarenessContext',
202
+ 'user32.dll',
203
+ $midsceneMethodAttributes,
204
+ [System.Reflection.CallingConventions]::Standard,
205
+ [System.IntPtr],
206
+ [System.Type[]]@([System.IntPtr]),
207
+ [System.Runtime.InteropServices.CallingConvention]::Winapi,
208
+ [System.Runtime.InteropServices.CharSet]::None
209
+ )
210
+ $midsceneSetDpiMethod.SetImplementationFlags(
211
+ $midsceneSetDpiMethod.GetMethodImplementationFlags() -bor
212
+ [System.Reflection.MethodImplAttributes]::PreserveSig
213
+ )
214
+ $midsceneGetForegroundWindowMethod = $midsceneType.DefinePInvokeMethod(
215
+ 'GetForegroundWindow',
216
+ 'user32.dll',
217
+ $midsceneMethodAttributes,
218
+ [System.Reflection.CallingConventions]::Standard,
219
+ [System.IntPtr],
220
+ [System.Type[]]@(),
221
+ [System.Runtime.InteropServices.CallingConvention]::Winapi,
222
+ [System.Runtime.InteropServices.CharSet]::None
223
+ )
224
+ $midsceneGetForegroundWindowMethod.SetImplementationFlags(
225
+ $midsceneGetForegroundWindowMethod.GetMethodImplementationFlags() -bor
226
+ [System.Reflection.MethodImplAttributes]::PreserveSig
227
+ )
228
+ $midsceneGetWindowRectMethod = $midsceneType.DefinePInvokeMethod(
229
+ 'GetWindowRect',
230
+ 'user32.dll',
231
+ $midsceneMethodAttributes,
232
+ [System.Reflection.CallingConventions]::Standard,
233
+ [bool],
234
+ [System.Type[]]@([System.IntPtr], [System.IntPtr]),
235
+ [System.Runtime.InteropServices.CallingConvention]::Winapi,
236
+ [System.Runtime.InteropServices.CharSet]::None
237
+ )
238
+ $midsceneGetWindowRectMethod.SetImplementationFlags(
239
+ $midsceneGetWindowRectMethod.GetMethodImplementationFlags() -bor
240
+ [System.Reflection.MethodImplAttributes]::PreserveSig
241
+ )
242
+ $midsceneNativeMethods = $midsceneType.CreateType()
243
+ $midscenePerMonitorV2 = [System.IntPtr](-4)
244
+ $midscenePreviousDpiContext =
245
+ $midsceneNativeMethods::SetThreadDpiAwarenessContext($midscenePerMonitorV2)
246
+ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
247
+ throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
248
+ }
249
+ `.trim();
250
+ function runWindowsPhysicalPixelPowershell(script) {
251
+ const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
252
+ $ErrorActionPreference = 'Stop'
253
+ ${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
254
+ ${script}`;
255
+ const encoded = Buffer.from(physicalPixelScript, 'utf16le').toString('base64');
256
+ return execFileSync('powershell.exe', [
257
+ '-NoProfile',
258
+ '-NonInteractive',
259
+ '-EncodedCommand',
260
+ encoded
261
+ ], {
262
+ encoding: 'utf8',
263
+ timeout: POWERSHELL_TIMEOUT_MS,
264
+ maxBuffer: POWERSHELL_MAX_BUFFER,
265
+ windowsHide: true
266
+ });
267
+ }
268
+ function windows_pointer_define_property(obj, key, value) {
269
+ if (key in obj) Object.defineProperty(obj, key, {
270
+ value: value,
271
+ enumerable: true,
272
+ configurable: true,
273
+ writable: true
274
+ });
275
+ else obj[key] = value;
276
+ return obj;
277
+ }
278
+ const WINDOWS_POINTER_TOLERANCE_PX = 5;
279
+ function assertFinitePoint(point, context) {
280
+ if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) throw new Error(`${context} must contain finite coordinates, got (${point.x}, ${point.y})`);
281
+ }
282
+ function parseWindowsPointerPosition(output, context) {
283
+ const match = output.trim().match(/^(-?\d+),(-?\d+)$/);
284
+ if (!match) throw new Error(`${context} returned an invalid cursor position: ${JSON.stringify(output.trim())}`);
285
+ return {
286
+ x: Number(match[1]),
287
+ y: Number(match[2])
288
+ };
289
+ }
290
+ function windowsPointerPositionScript() {
291
+ return `
292
+ Add-Type -AssemblyName System.Windows.Forms, System.Drawing
293
+ $position = [System.Windows.Forms.Cursor]::Position
294
+ [Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
295
+ `.trim();
296
+ }
297
+ function windowsPointerMoveScript(point, options) {
298
+ assertFinitePoint(point, 'Windows pointer target');
299
+ if (options?.smoothSteps !== void 0 && !Number.isFinite(options.smoothSteps) || options?.smoothDelayMs !== void 0 && !Number.isFinite(options.smoothDelayMs)) throw new Error('Windows pointer smoothing options must be finite numbers');
300
+ const targetX = Math.round(point.x);
301
+ const targetY = Math.round(point.y);
302
+ const smoothSteps = Math.max(1, Math.round(options?.smoothSteps ?? 1));
303
+ const smoothDelayMs = Math.max(0, Math.round(options?.smoothDelayMs ?? 0));
304
+ return `
305
+ Add-Type -AssemblyName System.Windows.Forms, System.Drawing
306
+ $targetX = ${targetX}
307
+ $targetY = ${targetY}
308
+ $smoothSteps = ${smoothSteps}
309
+ $smoothDelayMs = ${smoothDelayMs}
310
+ $start = [System.Windows.Forms.Cursor]::Position
311
+ for ($step = 1; $step -le $smoothSteps; $step += 1) {
312
+ $x = [int][Math]::Round($start.X + (($targetX - $start.X) * $step / [double]$smoothSteps))
313
+ $y = [int][Math]::Round($start.Y + (($targetY - $start.Y) * $step / [double]$smoothSteps))
314
+ [System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($x, $y)
315
+ if ($smoothDelayMs -gt 0 -and $step -lt $smoothSteps) {
316
+ Start-Sleep -Milliseconds $smoothDelayMs
317
+ }
318
+ }
319
+ [System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($targetX, $targetY)
320
+ $position = [System.Windows.Forms.Cursor]::Position
321
+ [Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
322
+ `.trim();
323
+ }
324
+ function parseWindowsWindowRect(output) {
325
+ const value = output.trim();
326
+ if (!value) return null;
327
+ const match = value.match(/^(-?\d+),(-?\d+),(\d+),(\d+)$/);
328
+ if (!match) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
329
+ const rect = {
330
+ x: Number(match[1]),
331
+ y: Number(match[2]),
332
+ width: Number(match[3]),
333
+ height: Number(match[4])
334
+ };
335
+ if (rect.width <= 0 || rect.height <= 0) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
336
+ return rect;
337
+ }
338
+ function windowsActiveWindowRectScript() {
339
+ return `
340
+ $midsceneWindowHandle = $midsceneNativeMethods::GetForegroundWindow()
341
+ if ($midsceneWindowHandle -eq [System.IntPtr]::Zero) {
342
+ return
343
+ }
344
+ $midsceneWindowRectBuffer = [System.Runtime.InteropServices.Marshal]::AllocHGlobal(16)
345
+ try {
346
+ if (-not $midsceneNativeMethods::GetWindowRect($midsceneWindowHandle, $midsceneWindowRectBuffer)) {
347
+ throw 'GetWindowRect failed for the foreground window.'
348
+ }
349
+ $left = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 0)
350
+ $top = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 4)
351
+ $right = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 8)
352
+ $bottom = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 12)
353
+ [Console]::Out.Write(('{0},{1},{2},{3}' -f $left, $top, ($right - $left), ($bottom - $top)))
354
+ }
355
+ finally {
356
+ [System.Runtime.InteropServices.Marshal]::FreeHGlobal($midsceneWindowRectBuffer)
357
+ }
358
+ `.trim();
359
+ }
360
+ class WindowsPointerDriver {
361
+ getPosition() {
362
+ return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerPositionScript()), 'Windows pointer query');
363
+ }
364
+ getActiveWindowRect() {
365
+ return parseWindowsWindowRect(this.options.runPhysicalPixelPowershell(windowsActiveWindowRectScript()));
366
+ }
367
+ moveTo(point, options) {
368
+ return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerMoveScript(point, options)), 'Windows pointer move');
369
+ }
370
+ constructor(options){
371
+ windows_pointer_define_property(this, "options", void 0);
372
+ this.options = options;
373
+ }
374
+ }
375
+ function windowsPointerDrift(expected, actual) {
376
+ return {
377
+ x: actual.x - expected.x,
378
+ y: actual.y - expected.y
379
+ };
380
+ }
381
+ function windowsPointerIsWithinTolerance(drift) {
382
+ return Math.abs(drift.x) <= WINDOWS_POINTER_TOLERANCE_PX && Math.abs(drift.y) <= WINDOWS_POINTER_TOLERANCE_PX;
383
+ }
187
384
  const debugXvfb = getDebug('computer:xvfb');
188
385
  const xvfbCleanupMonitorScript = String.raw`
189
386
  const parentPid = Number(process.argv[1]);
@@ -441,39 +638,36 @@ function sendKeyViaAppleScript(key, modifiers = []) {
441
638
  script
442
639
  ]);
443
640
  }
444
- const POWERSHELL_TIMEOUT_MS = 15000;
445
- const POWERSHELL_MAX_BUFFER = 67108864;
446
641
  function escapePowershellSingleQuoted(value) {
447
642
  return value.replace(/'/g, "''");
448
643
  }
449
- function runPowershell(script) {
450
- const prefixed = `$ProgressPreference = 'SilentlyContinue'\n${script}`;
451
- const encoded = Buffer.from(prefixed, 'utf16le').toString('base64');
452
- return execFileSync('powershell.exe', [
453
- '-NoProfile',
454
- '-NonInteractive',
455
- '-EncodedCommand',
456
- encoded
457
- ], {
458
- encoding: 'utf8',
459
- timeout: POWERSHELL_TIMEOUT_MS,
460
- maxBuffer: POWERSHELL_MAX_BUFFER,
461
- windowsHide: true
462
- });
463
- }
464
- function listWindowsDisplays() {
644
+ function readWindowsDisplayGeometries() {
465
645
  const script = `
466
646
  Add-Type -AssemblyName System.Windows.Forms
467
647
  $s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
468
- [PSCustomObject]@{ id = $_.DeviceName; name = $_.DeviceName; primary = $_.Primary }
648
+ $b = $_.Bounds
649
+ [PSCustomObject]@{
650
+ id = $_.DeviceName
651
+ name = $_.DeviceName
652
+ primary = $_.Primary
653
+ bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
654
+ }
469
655
  }
470
656
  ConvertTo-Json @($s) -Compress
471
657
  `.trim();
472
- const parsed = JSON.parse(runPowershell(script).trim());
473
- return parsed.map((d)=>({
474
- id: String(d.id),
475
- name: d.name || String(d.id),
476
- primary: d.primary || false
658
+ const output = runWindowsPhysicalPixelPowershell(script).trim();
659
+ if (!output) throw new Error('Windows display enumeration returned no data');
660
+ const parsed = JSON.parse(output);
661
+ if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
662
+ const displays = parsed.filter(isWindowsDisplayGeometry);
663
+ if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
664
+ return displays;
665
+ }
666
+ function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
667
+ return geometries.map((display)=>({
668
+ id: display.id,
669
+ name: display.name,
670
+ primary: display.primary
477
671
  }));
478
672
  }
479
673
  let device_libnut = null;
@@ -575,6 +769,16 @@ function getDisplayInfoBinary() {
575
769
  function isFiniteNumber(value) {
576
770
  return 'number' == typeof value && Number.isFinite(value);
577
771
  }
772
+ function isDisplayBounds(value) {
773
+ if (!value || 'object' != typeof value) return false;
774
+ const bounds = value;
775
+ return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
776
+ }
777
+ function isWindowsDisplayGeometry(value) {
778
+ if (!value || 'object' != typeof value) return false;
779
+ const candidate = value;
780
+ return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
781
+ }
578
782
  function isDarwinDisplayGeometry(value) {
579
783
  if (!value || 'object' != typeof value) return false;
580
784
  const candidate = value;
@@ -617,9 +821,11 @@ function readDarwinFrontmostApplication() {
617
821
  return;
618
822
  }
619
823
  }
620
- async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
621
- await inputDriver.delay(CLICK_SETTLE_DELAY);
622
- const current = inputDriver.getMousePos();
824
+ async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, current, holdDuration, reason) {
825
+ const drift = {
826
+ x: current.x - targetX,
827
+ y: current.y - targetY
828
+ };
623
829
  debugComputerInput('tap mouse moved %o', {
624
830
  reason,
625
831
  target: {
@@ -627,10 +833,7 @@ async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDurati
627
833
  y: targetY
628
834
  },
629
835
  current,
630
- drift: {
631
- x: current.x - targetX,
632
- y: current.y - targetY
633
- }
836
+ drift
634
837
  });
635
838
  await inputDriver.withMouseButton('left', async ()=>{
636
839
  debugComputerInput('tap mouse down %o', {
@@ -649,9 +852,18 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
649
852
  if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
650
853
  return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
651
854
  }
652
- function resolveDisplayGeometry(displayId) {
653
- if ('darwin' !== process.platform) return;
654
- return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
855
+ function resolveWindowsDisplayGeometryFromList(displayId, displays) {
856
+ if (!displays.length) return;
857
+ if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
858
+ return displays.find((display)=>display.id === displayId);
859
+ }
860
+ function resolveDisplayGeometry(displayId, windowsDisplays) {
861
+ if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
862
+ if ('win32' === process.platform) {
863
+ const geometry = resolveWindowsDisplayGeometryFromList(displayId, windowsDisplays ?? readWindowsDisplayGeometries());
864
+ if (!geometry) throw new Error(displayId ? `Requested Windows display not found: ${displayId}` : 'No Windows displays were detected');
865
+ return geometry;
866
+ }
655
867
  }
656
868
  function mapDisplayLocalPointToGlobal(point, geometry) {
657
869
  if (!geometry) return point;
@@ -737,6 +949,30 @@ function normalizePrimaryKey(key) {
737
949
  return KEY_NAME_MAP[lowerKey] || lowerKey;
738
950
  }
739
951
  class ComputerDevice {
952
+ async moveGlobalPointer(point, context, smooth) {
953
+ if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
954
+ const target = {
955
+ x: Math.round(point.x),
956
+ y: Math.round(point.y)
957
+ };
958
+ if ('win32' === process.platform) {
959
+ const actual = this.windowsPointerDriver.moveTo(target, {
960
+ smoothSteps: smooth?.smoothSteps,
961
+ smoothDelayMs: smooth?.smoothDelay
962
+ });
963
+ const drift = windowsPointerDrift(target, actual);
964
+ if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`${context}: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
965
+ await this.inputDriver.delay(CLICK_SETTLE_DELAY);
966
+ return actual;
967
+ }
968
+ if (smooth) await this.inputDriver.smoothMoveMouse(target.x, target.y, smooth.smoothSteps, smooth.smoothDelay);
969
+ else this.inputDriver.moveMouse(target.x, target.y);
970
+ await this.inputDriver.delay(CLICK_SETTLE_DELAY);
971
+ return this.inputDriver.getMousePos();
972
+ }
973
+ moveDisplayPointer(point, context, smooth) {
974
+ return this.moveGlobalPointer(this.toGlobalPoint(point), context, smooth);
975
+ }
740
976
  async focusKeyboardTarget(element, delayMs) {
741
977
  const [x, y] = element.center;
742
978
  if ('darwin' === process.platform) await this.inputPrimitives.pointer.tap({
@@ -744,11 +980,10 @@ class ComputerDevice {
744
980
  y
745
981
  });
746
982
  else {
747
- const point = this.toGlobalPoint({
983
+ await this.moveDisplayPointer({
748
984
  x,
749
985
  y
750
- });
751
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
986
+ }, 'Mouse did not reach the keyboard focus target');
752
987
  this.inputDriver.mouseClick('left');
753
988
  }
754
989
  await this.inputDriver.delay(delayMs);
@@ -767,7 +1002,7 @@ class ComputerDevice {
767
1002
  }));
768
1003
  } catch (error) {
769
1004
  debugDevice(`Failed to list displays: ${error}`);
770
- return [];
1005
+ throw new Error(`Failed to list displays: ${error}`);
771
1006
  }
772
1007
  }
773
1008
  async connect() {
@@ -796,9 +1031,10 @@ class ComputerDevice {
796
1031
  }
797
1032
  }
798
1033
  device_libnut = await getLibnut();
799
- this.displayGeometry = resolveDisplayGeometry(this.displayId);
1034
+ const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
1035
+ this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
800
1036
  const size = await this.size();
801
- const displays = await ComputerDevice.listDisplays();
1037
+ const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
802
1038
  const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
803
1039
  this.description = `
804
1040
  Type: Computer
@@ -808,7 +1044,7 @@ Screen Size: ${size.width}x${size.height}
808
1044
  Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
809
1045
  `;
810
1046
  debugDevice('Computer device connected', this.description);
811
- await this.healthCheck();
1047
+ await this.healthCheck(displays);
812
1048
  } catch (error) {
813
1049
  if (this.xvfbInstance) {
814
1050
  if (!this.options?.keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
@@ -827,9 +1063,9 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
827
1063
  throw new Error(`Unable to connect to computer device: ${error}`);
828
1064
  }
829
1065
  }
830
- async healthCheck() {
1066
+ async healthCheck(displays) {
831
1067
  console.log('[HealthCheck] Starting health check...');
832
- console.log("[HealthCheck] @midscene/computer v1.12.3-beta-20260902081623.0");
1068
+ console.log("[HealthCheck] @midscene/computer v1.12.3");
833
1069
  console.log('[HealthCheck] Taking screenshot...');
834
1070
  const screenshotTimeout = 15000;
835
1071
  let timeoutId;
@@ -841,23 +1077,38 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
841
1077
  timeoutPromise
842
1078
  ]);
843
1079
  console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
844
- console.log('[HealthCheck] Moving mouse...');
845
- const startPos = this.inputDriver.getMousePos();
1080
+ console.log('[HealthCheck] Verifying mouse control...');
1081
+ const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
846
1082
  console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
847
- const offsetX = Math.floor(40 * Math.random()) + 10;
848
- const offsetY = Math.floor(40 * Math.random()) + 10;
849
- const targetX = startPos.x + offsetX;
850
- const targetY = startPos.y + offsetY;
851
- console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
852
- this.inputDriver.moveMouse(targetX, targetY);
853
- await sleep(50);
854
- const movedPos = this.inputDriver.getMousePos();
855
- console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
856
- const deltaX = Math.abs(movedPos.x - targetX);
857
- const deltaY = Math.abs(movedPos.y - targetY);
858
- if (deltaX > 5 || deltaY > 5) {
859
- const msg = `[HealthCheck] WARNING: Mouse control may not be working. Expected (${targetX}, ${targetY}), got (${movedPos.x}, ${movedPos.y}), delta=(${deltaX}, ${deltaY})`;
860
- warnDevice(msg);
1083
+ if ('win32' === process.platform) {
1084
+ if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
1085
+ const bounds = this.displayGeometry.bounds;
1086
+ const target = {
1087
+ x: Math.round(bounds.x + bounds.width / 2),
1088
+ y: Math.round(bounds.y + bounds.height / 2)
1089
+ };
1090
+ try {
1091
+ const actual = this.windowsPointerDriver.moveTo(target);
1092
+ const drift = windowsPointerDrift(target, actual);
1093
+ if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`Windows screenshot-space pointer verification: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
1094
+ } finally{
1095
+ this.windowsPointerDriver.moveTo(startPos);
1096
+ }
1097
+ console.log(`[HealthCheck] Mouse verified in screenshot-space display bounds (${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`);
1098
+ } else {
1099
+ const offsetX = Math.floor(40 * Math.random()) + 10;
1100
+ const offsetY = Math.floor(40 * Math.random()) + 10;
1101
+ const targetX = startPos.x + offsetX;
1102
+ const targetY = startPos.y + offsetY;
1103
+ console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
1104
+ try {
1105
+ this.inputDriver.moveMouse(targetX, targetY);
1106
+ await sleep(CLICK_SETTLE_DELAY);
1107
+ this.inputDriver.assertMousePosition(targetX, targetY, 'Mouse health check failed');
1108
+ } finally{
1109
+ this.inputDriver.moveMouse(startPos.x, startPos.y);
1110
+ console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
1111
+ }
861
1112
  }
862
1113
  if ('win32' === process.platform && !this.isRunningAsAdmin()) {
863
1114
  const hint = [
@@ -867,10 +1118,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
867
1118
  ].join(' ');
868
1119
  warnDevice(`[HealthCheck] ${hint}`);
869
1120
  }
870
- this.inputDriver.moveMouse(startPos.x, startPos.y);
871
- console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
872
1121
  console.log('[HealthCheck] Listing monitors...');
873
- const displays = await ComputerDevice.listDisplays();
874
1122
  if (displays.length > 0) {
875
1123
  console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
876
1124
  for (const display of displays){
@@ -938,7 +1186,6 @@ Original error: ${lastRawMessage}`);
938
1186
  $screen = [System.Windows.Forms.Screen]::AllScreens | Where-Object { $_.DeviceName -eq $dn } | Select-Object -First 1
939
1187
  if (-not $screen) { throw "Requested display not found: $dn" }` : '$screen = [System.Windows.Forms.Screen]::PrimaryScreen';
940
1188
  const script = `
941
- $ErrorActionPreference = 'Stop'
942
1189
  Add-Type -AssemblyName System.Windows.Forms, System.Drawing
943
1190
  ${selectScreen}
944
1191
  $b = $screen.Bounds
@@ -952,7 +1199,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
952
1199
  `.trim();
953
1200
  let stdout;
954
1201
  try {
955
- stdout = runPowershell(script);
1202
+ stdout = runWindowsPhysicalPixelPowershell(script);
956
1203
  } catch (error) {
957
1204
  const message = error instanceof Error ? error.message : String(error);
958
1205
  throw new Error(`Failed to take screenshot on Windows: ${message}`);
@@ -1054,11 +1301,15 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
1054
1301
  }
1055
1302
  resolveUntargetedScrollPoint(screenSize) {
1056
1303
  if ('win32' === process.platform) {
1057
- const activeWindowRect = this.inputDriver.getActiveWindowRect();
1058
- if (activeWindowRect) return {
1059
- x: activeWindowRect.x + activeWindowRect.width / 2,
1060
- y: activeWindowRect.y + activeWindowRect.height / 2
1061
- };
1304
+ const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
1305
+ if (activeWindowRect) {
1306
+ const activeWindowCenter = {
1307
+ x: activeWindowRect.x + activeWindowRect.width / 2,
1308
+ y: activeWindowRect.y + activeWindowRect.height / 2
1309
+ };
1310
+ const bounds = this.displayGeometry?.bounds;
1311
+ if (!bounds || activeWindowCenter.x >= bounds.x && activeWindowCenter.x < bounds.x + bounds.width && activeWindowCenter.y >= bounds.y && activeWindowCenter.y < bounds.y + bounds.height) return activeWindowCenter;
1312
+ }
1062
1313
  }
1063
1314
  return this.toGlobalPoint({
1064
1315
  x: screenSize.width / 2,
@@ -1069,17 +1320,16 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
1069
1320
  if (param.locate) {
1070
1321
  const element = param.locate;
1071
1322
  const [x, y] = element.center;
1072
- const point = this.toGlobalPoint({
1323
+ await this.moveDisplayPointer({
1073
1324
  x,
1074
1325
  y
1075
- });
1076
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
1326
+ }, 'Mouse did not reach the scroll target');
1077
1327
  return;
1078
1328
  }
1079
1329
  const screenSize = await this.size();
1080
1330
  if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
1081
1331
  const point = this.resolveUntargetedScrollPoint(screenSize);
1082
- this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
1332
+ await this.moveGlobalPointer(point, 'Mouse did not reach the scroll viewport');
1083
1333
  await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
1084
1334
  return screenSize;
1085
1335
  }
@@ -1204,6 +1454,9 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
1204
1454
  runPhasedScroll,
1205
1455
  debug: (message)=>debugDevice(message)
1206
1456
  }));
1457
+ device_define_property(this, "windowsPointerDriver", new WindowsPointerDriver({
1458
+ runPhysicalPixelPowershell: runWindowsPhysicalPixelPowershell
1459
+ }));
1207
1460
  device_define_property(this, "useAppleScript", void 0);
1208
1461
  device_define_property(this, "adminCheckCache", void 0);
1209
1462
  device_define_property(this, "uri", void 0);
@@ -1228,15 +1481,17 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
1228
1481
  },
1229
1482
  holdDuration,
1230
1483
  displayId: this.displayId,
1231
- displayGeometry: this.displayGeometry ? {
1232
- screenIndex: this.displayGeometry.screenIndex,
1233
- cgDisplayId: this.displayGeometry.cgDisplayId,
1234
- bounds: this.displayGeometry.bounds
1235
- } : void 0
1484
+ displayGeometry: this.displayGeometry
1236
1485
  });
1237
1486
  const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
1238
- await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
1239
- await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'primary');
1487
+ const current = await this.moveGlobalPointer({
1488
+ x: targetX,
1489
+ y: targetY
1490
+ }, 'Mouse did not reach the tap target', {
1491
+ smoothSteps: SMOOTH_MOVE_STEPS_TAP,
1492
+ smoothDelay: SMOOTH_MOVE_DELAY_TAP
1493
+ });
1494
+ await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, current, holdDuration, 'primary');
1240
1495
  if (frontmostBefore && 'darwin' === process.platform) {
1241
1496
  await sleep(CLICK_FOCUS_SETTLE_DELAY);
1242
1497
  const frontmostAfter = readDarwinFrontmostApplication();
@@ -1247,42 +1502,43 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
1247
1502
  focusChanged
1248
1503
  });
1249
1504
  if (focusChanged) {
1250
- this.inputDriver.moveMouse(targetX, targetY);
1251
- await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'focus-follow-up');
1505
+ const followUpCurrent = await this.moveGlobalPointer({
1506
+ x: targetX,
1507
+ y: targetY
1508
+ }, 'Mouse did not reach the focus follow-up target');
1509
+ await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, followUpCurrent, holdDuration, 'focus-follow-up');
1252
1510
  }
1253
1511
  }
1254
1512
  },
1255
1513
  doubleClick: async ({ x, y })=>{
1256
- const target = this.toGlobalPoint({
1514
+ await this.moveDisplayPointer({
1257
1515
  x,
1258
1516
  y
1259
- });
1260
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1517
+ }, 'Mouse did not reach the double-click target');
1261
1518
  this.inputDriver.mouseClick('left', true);
1262
1519
  },
1263
1520
  rightClick: async ({ x, y })=>{
1264
- const target = this.toGlobalPoint({
1521
+ await this.moveDisplayPointer({
1265
1522
  x,
1266
1523
  y
1267
- });
1268
- this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
1524
+ }, 'Mouse did not reach the right-click target');
1269
1525
  this.inputDriver.mouseClick('right');
1270
1526
  },
1271
1527
  hover: async ({ x, y })=>{
1272
- const target = this.toGlobalPoint({
1528
+ await this.moveDisplayPointer({
1273
1529
  x,
1274
1530
  y
1531
+ }, 'Mouse did not reach the hover target', {
1532
+ smoothSteps: SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1533
+ smoothDelay: SMOOTH_MOVE_DELAY_MOUSE_MOVE
1275
1534
  });
1276
- await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
1277
1535
  await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
1278
1536
  },
1279
1537
  dragAndDrop: async (from, to)=>{
1280
- const globalFrom = this.toGlobalPoint(from);
1281
- const globalTo = this.toGlobalPoint(to);
1282
- this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
1538
+ await this.moveDisplayPointer(from, 'Mouse did not reach the drag start target');
1283
1539
  await this.inputDriver.withMouseButton('left', async ()=>{
1284
1540
  await this.inputDriver.delay(100);
1285
- this.inputDriver.moveMouse(Math.round(globalTo.x), Math.round(globalTo.y));
1541
+ await this.moveDisplayPointer(to, 'Mouse did not reach the drag end target');
1286
1542
  await this.inputDriver.delay(100);
1287
1543
  });
1288
1544
  }
@@ -2255,7 +2511,7 @@ const tools = new ComputerMidsceneTools({
2255
2511
  });
2256
2512
  runToolsCLI(tools, 'midscene-computer', {
2257
2513
  stripPrefix: 'computer_',
2258
- version: "1.12.3-beta-20260902081623.0",
2514
+ version: "1.12.3",
2259
2515
  extraCommands: createReportCliCommands()
2260
2516
  }).catch((e)=>{
2261
2517
  process.exit(reportCLIError(e));