@midscene/computer 1.17.7 → 1.17.8-beta-20261110095715.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2782 @@
1
+ import { execFileSync, execSync, spawn, spawnSync } from "node:child_process";
2
+ import { randomUUID } from "node:crypto";
3
+ import { chmodSync, existsSync, readFileSync, rmSync, statSync } from "node:fs";
4
+ import { createRequire } from "node:module";
5
+ import { tmpdir } from "node:os";
6
+ import { dirname, join, resolve as external_node_path_resolve } from "node:path";
7
+ import { fileURLToPath } from "node:url";
8
+ import { defineAction, defineActionsFromInputPrimitives, inputStrategies, resolveTextInputOptions, sendTextSequentially, shouldInputSequentially } from "@midscene/core/device";
9
+ import { sleep } from "@midscene/core/utils";
10
+ import { createImgBase64ByFormat } from "@midscene/shared/img";
11
+ import { getDebug } from "@midscene/shared/logger";
12
+ import screenshot_desktop from "screenshot-desktop";
13
+ import node_assert from "node:assert";
14
+ import { hasActiveCliInterruptWaiter } from "@midscene/shared/cli/interrupt";
15
+ import { Agent } from "@midscene/core/agent";
16
+ import { once } from "node:events";
17
+ import { createInterface } from "node:readline";
18
+ import { createReportCliCommands, z } from "@midscene/core";
19
+ import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/agent-tools/agent-behavior-init-args";
20
+ import { BaseMidsceneTools } from "@midscene/shared/agent-tools/base-tools";
21
+ const debugKeyboard = getDebug('computer:keyboard');
22
+ const APPLE_SCRIPT_KEY_CODES = {
23
+ return: 36,
24
+ enter: 36,
25
+ tab: 48,
26
+ space: 49,
27
+ backspace: 51,
28
+ delete: 51,
29
+ escape: 53,
30
+ forwarddelete: 117,
31
+ left: 123,
32
+ right: 124,
33
+ down: 125,
34
+ up: 126,
35
+ home: 115,
36
+ end: 119,
37
+ pageup: 116,
38
+ pagedown: 121,
39
+ f1: 122,
40
+ f2: 120,
41
+ f3: 99,
42
+ f4: 118,
43
+ f5: 96,
44
+ f6: 97,
45
+ f7: 98,
46
+ f8: 100,
47
+ f9: 101,
48
+ f10: 109,
49
+ f11: 103,
50
+ f12: 111
51
+ };
52
+ const APPLE_SCRIPT_MODIFIER_KEYS = {
53
+ command: 'command',
54
+ cmd: 'command',
55
+ control: 'control',
56
+ ctrl: 'control',
57
+ shift: 'shift',
58
+ alt: 'option',
59
+ option: 'option',
60
+ meta: 'command'
61
+ };
62
+ function buildKeyCommand(key) {
63
+ const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
64
+ if (void 0 !== keyCode) return `key code ${keyCode}`;
65
+ const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
66
+ return `keystroke "${escapedKey}"`;
67
+ }
68
+ function resolveModifierKeys(modifiers) {
69
+ return modifiers.map((modifier)=>APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]).filter((modifier)=>void 0 !== modifier);
70
+ }
71
+ function buildAppleScriptKeyPress(key, modifiers = []) {
72
+ const modifierKeys = resolveModifierKeys(modifiers);
73
+ const modifierClause = modifierKeys.length ? ` using {${modifierKeys.map((modifier)=>`${modifier} down`).join(', ')}}` : '';
74
+ return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
75
+ }
76
+ function sendKeyViaAppleScript(key, modifiers = []) {
77
+ const script = buildAppleScriptKeyPress(key, modifiers);
78
+ debugKeyboard('sendKeyViaAppleScript', {
79
+ key,
80
+ modifiers,
81
+ script
82
+ });
83
+ execFileSync("osascript", [
84
+ '-e',
85
+ script
86
+ ]);
87
+ }
88
+ function _define_property(obj, key, value) {
89
+ if (key in obj) Object.defineProperty(obj, key, {
90
+ value: value,
91
+ enumerable: true,
92
+ configurable: true,
93
+ writable: true
94
+ });
95
+ else obj[key] = value;
96
+ return obj;
97
+ }
98
+ const MOUSE_COORDINATE_TOLERANCE_PX = 5;
99
+ class ComputerInputDriver {
100
+ destroy() {
101
+ if (this.destroyed) return;
102
+ this.destroyed = true;
103
+ this.rejectPendingInputDelays();
104
+ }
105
+ getScreenSize() {
106
+ return this.getLibnutOrThrow('getScreenSize').getScreenSize();
107
+ }
108
+ getMousePos() {
109
+ return this.getLibnutOrThrow('getMousePos').getMousePos();
110
+ }
111
+ moveMouse(x, y) {
112
+ this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
113
+ }
114
+ assertMousePosition(targetX, targetY, context) {
115
+ const current = this.getMousePos();
116
+ const drift = {
117
+ x: current.x - targetX,
118
+ y: current.y - targetY
119
+ };
120
+ if (Math.abs(drift.x) > MOUSE_COORDINATE_TOLERANCE_PX || Math.abs(drift.y) > MOUSE_COORDINATE_TOLERANCE_PX) throw new Error(`${context}: expected (${targetX}, ${targetY}), got (${current.x}, ${current.y}), drift=(${drift.x}, ${drift.y})`);
121
+ return drift;
122
+ }
123
+ focusActiveWindow() {
124
+ const lib = this.getLibnutOrThrow('focusActiveWindow');
125
+ if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
126
+ try {
127
+ const handle = lib.getActiveWindow();
128
+ if (!handle) return false;
129
+ lib.focusWindow(handle);
130
+ return true;
131
+ } catch (error) {
132
+ this.options.debug(`focusActiveWindow failed: ${error}`);
133
+ return false;
134
+ }
135
+ }
136
+ mouseClick(button, double) {
137
+ const lib = this.getLibnutOrThrow('mouseClick');
138
+ if (void 0 !== double) lib.mouseClick(button, double);
139
+ else if (void 0 !== button) lib.mouseClick(button);
140
+ else lib.mouseClick();
141
+ }
142
+ mouseToggle(state, button = 'left') {
143
+ this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
144
+ }
145
+ scrollMouse(x, y) {
146
+ this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
147
+ }
148
+ async emitScrollDetents(dx, dy, detents, delayMs) {
149
+ this.assertActive('emitScrollDetents');
150
+ for(let i = 0; i < detents; i++){
151
+ this.scrollMouse(dx, dy);
152
+ if (i < detents - 1) await this.delay(delayMs);
153
+ }
154
+ }
155
+ keyTap(key, modifiers) {
156
+ const lib = this.getLibnutOrThrow('keyTap');
157
+ if (void 0 !== modifiers) lib.keyTap(key, modifiers);
158
+ else lib.keyTap(key);
159
+ }
160
+ keyToggle(key, state, modifiers) {
161
+ const lib = this.getLibnutOrThrow('keyToggle');
162
+ if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
163
+ else lib.keyToggle(key, state);
164
+ }
165
+ async keyTapWithModifierDelay(key, modifiers, delayMs) {
166
+ const uniqueModifiers = [
167
+ ...new Set(modifiers)
168
+ ];
169
+ if (0 === uniqueModifiers.length) return void this.keyTap(key);
170
+ if (0 === delayMs) return void this.keyTap(key, uniqueModifiers);
171
+ const pressedModifiers = [];
172
+ let failure;
173
+ try {
174
+ for (const modifier of uniqueModifiers){
175
+ this.keyToggle(modifier, 'down');
176
+ pressedModifiers.push(modifier);
177
+ await this.delay(delayMs);
178
+ }
179
+ this.keyTap(key);
180
+ await this.delay(delayMs);
181
+ } catch (error) {
182
+ failure = error;
183
+ }
184
+ for (const modifier of pressedModifiers.reverse()){
185
+ const releaseFailure = this.tryReleaseKey(modifier);
186
+ if (void 0 === failure && void 0 !== releaseFailure) failure = releaseFailure;
187
+ if (void 0 === failure) try {
188
+ await this.delay(delayMs);
189
+ } catch (error) {
190
+ failure = error;
191
+ }
192
+ }
193
+ if (void 0 !== failure) throw failure;
194
+ }
195
+ typeString(text) {
196
+ this.getLibnutOrThrow('typeString').typeString(text);
197
+ }
198
+ sendKeyViaAppleScript(key, modifiers = []) {
199
+ this.assertActive('sendKeyViaAppleScript');
200
+ this.options.sendKeyViaAppleScript(key, modifiers);
201
+ }
202
+ sendKey(key, modifiers = []) {
203
+ if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
204
+ if (modifiers.length > 0) this.keyTap(key, modifiers);
205
+ else this.keyTap(key);
206
+ }
207
+ runPhasedScroll(direction, pixels, steps) {
208
+ this.assertActive('runPhasedScroll');
209
+ return this.options.runPhasedScroll(direction, pixels, steps);
210
+ }
211
+ async delay(ms) {
212
+ this.assertActive('delay');
213
+ return new Promise((resolve, reject)=>{
214
+ const waitRef = {
215
+ timeoutId: setTimeout(()=>{
216
+ this.pendingInputDelayWaits.delete(waitRef);
217
+ try {
218
+ this.assertActive('delay');
219
+ resolve();
220
+ } catch (error) {
221
+ reject(error);
222
+ }
223
+ }, ms),
224
+ reject
225
+ };
226
+ this.pendingInputDelayWaits.add(waitRef);
227
+ });
228
+ }
229
+ async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
230
+ const currentPos = this.getMousePos();
231
+ for(let i = 1; i <= steps; i++){
232
+ const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
233
+ const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
234
+ this.moveMouse(stepX, stepY);
235
+ await this.delay(stepDelay);
236
+ }
237
+ }
238
+ async withMouseButton(button, run) {
239
+ this.mouseToggle('down', button);
240
+ try {
241
+ return await run();
242
+ } finally{
243
+ this.releaseMouseButton(button);
244
+ }
245
+ }
246
+ getLibnutOrThrow(methodName) {
247
+ this.assertActive(methodName);
248
+ const libnut = this.options.getLibnut();
249
+ node_assert(libnut, 'libnut not initialized');
250
+ return libnut;
251
+ }
252
+ assertActive(methodName) {
253
+ if (this.destroyed) throw this.createDestroyedError(methodName);
254
+ }
255
+ createDestroyedError(methodName) {
256
+ return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
257
+ }
258
+ releaseMouseButton(button) {
259
+ try {
260
+ const libnut = this.options.getLibnut();
261
+ node_assert(libnut, 'libnut not initialized');
262
+ libnut.mouseToggle('up', button);
263
+ } catch (error) {
264
+ this.options.debug(`Failed to release mouse button ${button}: ${error}`);
265
+ }
266
+ }
267
+ tryReleaseKey(key) {
268
+ try {
269
+ const libnut = this.options.getLibnut();
270
+ node_assert(libnut, 'libnut not initialized');
271
+ libnut.keyToggle(key, 'up');
272
+ return;
273
+ } catch (error) {
274
+ const releaseError = new Error(`Failed to release modifier key "${key}"`, {
275
+ cause: error
276
+ });
277
+ this.options.debug(`${releaseError.message}: ${error}`);
278
+ return releaseError;
279
+ }
280
+ }
281
+ rejectPendingInputDelays() {
282
+ const error = this.createDestroyedError('in-flight input');
283
+ for (const waitRef of this.pendingInputDelayWaits){
284
+ clearTimeout(waitRef.timeoutId);
285
+ waitRef.reject(error);
286
+ }
287
+ this.pendingInputDelayWaits.clear();
288
+ }
289
+ constructor(options){
290
+ _define_property(this, "options", void 0);
291
+ _define_property(this, "destroyed", void 0);
292
+ _define_property(this, "pendingInputDelayWaits", void 0);
293
+ this.options = options;
294
+ this.destroyed = false;
295
+ this.pendingInputDelayWaits = new Set();
296
+ }
297
+ }
298
+ const US_SHIFTED_CHARACTER_KEYS = new Map([
299
+ [
300
+ '~',
301
+ '`'
302
+ ],
303
+ [
304
+ '!',
305
+ '1'
306
+ ],
307
+ [
308
+ '@',
309
+ '2'
310
+ ],
311
+ [
312
+ '#',
313
+ '3'
314
+ ],
315
+ [
316
+ '$',
317
+ '4'
318
+ ],
319
+ [
320
+ '%',
321
+ '5'
322
+ ],
323
+ [
324
+ '^',
325
+ '6'
326
+ ],
327
+ [
328
+ '&',
329
+ '7'
330
+ ],
331
+ [
332
+ '*',
333
+ '8'
334
+ ],
335
+ [
336
+ '(',
337
+ '9'
338
+ ],
339
+ [
340
+ ')',
341
+ '0'
342
+ ],
343
+ [
344
+ '_',
345
+ '-'
346
+ ],
347
+ [
348
+ '+',
349
+ '='
350
+ ],
351
+ [
352
+ '{',
353
+ '['
354
+ ],
355
+ [
356
+ '}',
357
+ ']'
358
+ ],
359
+ [
360
+ '|',
361
+ '\\'
362
+ ],
363
+ [
364
+ ':',
365
+ ';'
366
+ ],
367
+ [
368
+ '"',
369
+ "'"
370
+ ],
371
+ [
372
+ '<',
373
+ ','
374
+ ],
375
+ [
376
+ '>',
377
+ '.'
378
+ ],
379
+ [
380
+ '?',
381
+ '/'
382
+ ]
383
+ ]);
384
+ function resolveShiftedKey(character, keyboardLayout) {
385
+ if (/^[A-Z]$/.test(character)) return character.toLowerCase();
386
+ return 'en-US' === keyboardLayout ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
387
+ }
388
+ function clampPointerPointToSize(point, size) {
389
+ const maximumX = Math.max(0, Math.floor(size.width) - 1);
390
+ const maximumY = Math.max(0, Math.floor(size.height) - 1);
391
+ return {
392
+ x: Math.max(0, Math.min(point.x, maximumX)),
393
+ y: Math.max(0, Math.min(point.y, maximumY))
394
+ };
395
+ }
396
+ const POWERSHELL_TIMEOUT_MS = 15000;
397
+ const POWERSHELL_MAX_BUFFER = 67108864;
398
+ const WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE = `
399
+ $midsceneAssemblyName = New-Object System.Reflection.AssemblyName('MidsceneDpiNative')
400
+ $midsceneAssembly = [System.AppDomain]::CurrentDomain.DefineDynamicAssembly(
401
+ $midsceneAssemblyName,
402
+ [System.Reflection.Emit.AssemblyBuilderAccess]::Run
403
+ )
404
+ $midsceneModule = $midsceneAssembly.DefineDynamicModule('MidsceneDpiNativeModule')
405
+ $midsceneType = $midsceneModule.DefineType(
406
+ 'MidsceneDpiNative.User32',
407
+ [System.Reflection.TypeAttributes]'Public, Class, Sealed, Abstract'
408
+ )
409
+ $midsceneMethodAttributes =
410
+ [System.Reflection.MethodAttributes]::Public -bor
411
+ [System.Reflection.MethodAttributes]::Static -bor
412
+ [System.Reflection.MethodAttributes]::PinvokeImpl
413
+ $midsceneSetDpiMethod = $midsceneType.DefinePInvokeMethod(
414
+ 'SetThreadDpiAwarenessContext',
415
+ 'user32.dll',
416
+ $midsceneMethodAttributes,
417
+ [System.Reflection.CallingConventions]::Standard,
418
+ [System.IntPtr],
419
+ [System.Type[]]@([System.IntPtr]),
420
+ [System.Runtime.InteropServices.CallingConvention]::Winapi,
421
+ [System.Runtime.InteropServices.CharSet]::None
422
+ )
423
+ $midsceneSetDpiMethod.SetImplementationFlags(
424
+ $midsceneSetDpiMethod.GetMethodImplementationFlags() -bor
425
+ [System.Reflection.MethodImplAttributes]::PreserveSig
426
+ )
427
+ $midsceneGetForegroundWindowMethod = $midsceneType.DefinePInvokeMethod(
428
+ 'GetForegroundWindow',
429
+ 'user32.dll',
430
+ $midsceneMethodAttributes,
431
+ [System.Reflection.CallingConventions]::Standard,
432
+ [System.IntPtr],
433
+ [System.Type[]]@(),
434
+ [System.Runtime.InteropServices.CallingConvention]::Winapi,
435
+ [System.Runtime.InteropServices.CharSet]::None
436
+ )
437
+ $midsceneGetForegroundWindowMethod.SetImplementationFlags(
438
+ $midsceneGetForegroundWindowMethod.GetMethodImplementationFlags() -bor
439
+ [System.Reflection.MethodImplAttributes]::PreserveSig
440
+ )
441
+ $midsceneGetWindowRectMethod = $midsceneType.DefinePInvokeMethod(
442
+ 'GetWindowRect',
443
+ 'user32.dll',
444
+ $midsceneMethodAttributes,
445
+ [System.Reflection.CallingConventions]::Standard,
446
+ [bool],
447
+ [System.Type[]]@([System.IntPtr], [System.IntPtr]),
448
+ [System.Runtime.InteropServices.CallingConvention]::Winapi,
449
+ [System.Runtime.InteropServices.CharSet]::None
450
+ )
451
+ $midsceneGetWindowRectMethod.SetImplementationFlags(
452
+ $midsceneGetWindowRectMethod.GetMethodImplementationFlags() -bor
453
+ [System.Reflection.MethodImplAttributes]::PreserveSig
454
+ )
455
+ $midsceneNativeMethods = $midsceneType.CreateType()
456
+ $midscenePerMonitorV2 = [System.IntPtr](-4)
457
+ $midscenePreviousDpiContext =
458
+ $midsceneNativeMethods::SetThreadDpiAwarenessContext($midscenePerMonitorV2)
459
+ if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
460
+ throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
461
+ }
462
+ `.trim();
463
+ function runWindowsPhysicalPixelPowershell(script) {
464
+ const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
465
+ $ErrorActionPreference = 'Stop'
466
+ ${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
467
+ ${script}`;
468
+ return execFileSync('powershell.exe', [
469
+ '-NoProfile',
470
+ '-Command',
471
+ physicalPixelScript
472
+ ], {
473
+ encoding: 'utf8',
474
+ timeout: POWERSHELL_TIMEOUT_MS,
475
+ maxBuffer: POWERSHELL_MAX_BUFFER,
476
+ windowsHide: true
477
+ });
478
+ }
479
+ function isFiniteNumber(value) {
480
+ return 'number' == typeof value && Number.isFinite(value);
481
+ }
482
+ function isWindowsDisplayGeometry(value) {
483
+ if (!value || 'object' != typeof value) return false;
484
+ const candidate = value;
485
+ const bounds = candidate.bounds;
486
+ return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
487
+ }
488
+ function readWindowsDisplayGeometries() {
489
+ const script = `
490
+ Add-Type -AssemblyName System.Windows.Forms
491
+ $screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
492
+ $bounds = $_.Bounds
493
+ [PSCustomObject]@{
494
+ id = $_.DeviceName
495
+ name = $_.DeviceName
496
+ primary = $_.Primary
497
+ bounds = [PSCustomObject]@{
498
+ x = $bounds.X
499
+ y = $bounds.Y
500
+ width = $bounds.Width
501
+ height = $bounds.Height
502
+ }
503
+ }
504
+ })
505
+ [Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))
506
+ `.trim();
507
+ const output = runWindowsPhysicalPixelPowershell(script).trim();
508
+ if (!output) throw new Error('Windows display enumeration returned no data');
509
+ const parsed = JSON.parse(output);
510
+ if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
511
+ const displays = parsed.filter(isWindowsDisplayGeometry);
512
+ if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
513
+ if (0 === displays.length) throw new Error('Windows display enumeration returned no displays');
514
+ return displays;
515
+ }
516
+ function resolveWindowsDisplayGeometryFromList(displayId, displays) {
517
+ if (!displays.length) return;
518
+ if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
519
+ return displays.find((display)=>display.id === displayId);
520
+ }
521
+ function windows_pointer_define_property(obj, key, value) {
522
+ if (key in obj) Object.defineProperty(obj, key, {
523
+ value: value,
524
+ enumerable: true,
525
+ configurable: true,
526
+ writable: true
527
+ });
528
+ else obj[key] = value;
529
+ return obj;
530
+ }
531
+ const WINDOWS_POINTER_TOLERANCE_PX = 5;
532
+ function assertFinitePoint(point, context) {
533
+ if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) throw new Error(`${context} must contain finite coordinates, got (${point.x}, ${point.y})`);
534
+ }
535
+ function parseWindowsPointerPosition(output, context) {
536
+ const match = output.trim().match(/^(-?\d+),(-?\d+)$/);
537
+ if (!match) throw new Error(`${context} returned an invalid cursor position: ${JSON.stringify(output.trim())}`);
538
+ return {
539
+ x: Number(match[1]),
540
+ y: Number(match[2])
541
+ };
542
+ }
543
+ function windowsPointerPositionScript() {
544
+ return `
545
+ Add-Type -AssemblyName System.Windows.Forms, System.Drawing
546
+ $position = [System.Windows.Forms.Cursor]::Position
547
+ [Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
548
+ `.trim();
549
+ }
550
+ function windowsPointerMoveScript(point, options) {
551
+ assertFinitePoint(point, 'Windows pointer target');
552
+ if (options?.smoothSteps !== void 0 && !Number.isFinite(options.smoothSteps) || options?.smoothDelayMs !== void 0 && !Number.isFinite(options.smoothDelayMs)) throw new Error('Windows pointer smoothing options must be finite numbers');
553
+ const targetX = Math.round(point.x);
554
+ const targetY = Math.round(point.y);
555
+ const smoothSteps = Math.max(1, Math.round(options?.smoothSteps ?? 1));
556
+ const smoothDelayMs = Math.max(0, Math.round(options?.smoothDelayMs ?? 0));
557
+ return `
558
+ Add-Type -AssemblyName System.Windows.Forms, System.Drawing
559
+ $targetX = ${targetX}
560
+ $targetY = ${targetY}
561
+ $smoothSteps = ${smoothSteps}
562
+ $smoothDelayMs = ${smoothDelayMs}
563
+ $start = [System.Windows.Forms.Cursor]::Position
564
+ for ($step = 1; $step -le $smoothSteps; $step += 1) {
565
+ $x = [int][Math]::Round($start.X + (($targetX - $start.X) * $step / [double]$smoothSteps))
566
+ $y = [int][Math]::Round($start.Y + (($targetY - $start.Y) * $step / [double]$smoothSteps))
567
+ [System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($x, $y)
568
+ if ($smoothDelayMs -gt 0 -and $step -lt $smoothSteps) {
569
+ Start-Sleep -Milliseconds $smoothDelayMs
570
+ }
571
+ }
572
+ [System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($targetX, $targetY)
573
+ $position = [System.Windows.Forms.Cursor]::Position
574
+ [Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
575
+ `.trim();
576
+ }
577
+ function parseWindowsWindowRect(output) {
578
+ const value = output.trim();
579
+ if (!value) return null;
580
+ const match = value.match(/^(-?\d+),(-?\d+),(\d+),(\d+)$/);
581
+ if (!match) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
582
+ const rect = {
583
+ x: Number(match[1]),
584
+ y: Number(match[2]),
585
+ width: Number(match[3]),
586
+ height: Number(match[4])
587
+ };
588
+ if (rect.width <= 0 || rect.height <= 0) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
589
+ return rect;
590
+ }
591
+ function windowsActiveWindowRectScript() {
592
+ return `
593
+ $midsceneWindowHandle = $midsceneNativeMethods::GetForegroundWindow()
594
+ if ($midsceneWindowHandle -eq [System.IntPtr]::Zero) {
595
+ return
596
+ }
597
+ $midsceneWindowRectBuffer = [System.Runtime.InteropServices.Marshal]::AllocHGlobal(16)
598
+ try {
599
+ if (-not $midsceneNativeMethods::GetWindowRect($midsceneWindowHandle, $midsceneWindowRectBuffer)) {
600
+ throw 'GetWindowRect failed for the foreground window.'
601
+ }
602
+ $left = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 0)
603
+ $top = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 4)
604
+ $right = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 8)
605
+ $bottom = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 12)
606
+ [Console]::Out.Write(('{0},{1},{2},{3}' -f $left, $top, ($right - $left), ($bottom - $top)))
607
+ }
608
+ finally {
609
+ [System.Runtime.InteropServices.Marshal]::FreeHGlobal($midsceneWindowRectBuffer)
610
+ }
611
+ `.trim();
612
+ }
613
+ class WindowsPointerDriver {
614
+ getPosition() {
615
+ return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerPositionScript()), 'Windows pointer query');
616
+ }
617
+ getActiveWindowRect() {
618
+ return parseWindowsWindowRect(this.options.runPhysicalPixelPowershell(windowsActiveWindowRectScript()));
619
+ }
620
+ moveTo(point, options) {
621
+ return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerMoveScript(point, options)), 'Windows pointer move');
622
+ }
623
+ constructor(options){
624
+ windows_pointer_define_property(this, "options", void 0);
625
+ this.options = options;
626
+ }
627
+ }
628
+ function windowsPointerDrift(expected, actual) {
629
+ return {
630
+ x: actual.x - expected.x,
631
+ y: actual.y - expected.y
632
+ };
633
+ }
634
+ function windowsPointerIsWithinTolerance(drift) {
635
+ return Math.abs(drift.x) <= WINDOWS_POINTER_TOLERANCE_PX && Math.abs(drift.y) <= WINDOWS_POINTER_TOLERANCE_PX;
636
+ }
637
+ const debugXvfb = getDebug('computer:xvfb');
638
+ const xvfbCleanupMonitorScript = String.raw`
639
+ const parentPid = Number(process.argv[1]);
640
+ const xvfbPid = Number(process.argv[2]);
641
+ const timer = setInterval(() => {
642
+ try {
643
+ process.kill(xvfbPid, 0);
644
+ } catch {
645
+ clearInterval(timer);
646
+ process.exit(0);
647
+ }
648
+ try {
649
+ process.kill(parentPid, 0);
650
+ return;
651
+ } catch {
652
+ // The owner is gone, so its X11 clients can no longer receive XIO errors.
653
+ }
654
+ try {
655
+ process.kill(xvfbPid, 'SIGTERM');
656
+ } catch {
657
+ // Xvfb may have already exited.
658
+ }
659
+ clearInterval(timer);
660
+ }, 100);
661
+ `;
662
+ function scheduleXvfbStopAfterProcessExit(instance, parentPid = process.pid) {
663
+ const xvfbPid = instance.process.pid;
664
+ if (!xvfbPid) throw new Error('Cannot schedule Xvfb cleanup before its process starts');
665
+ const monitor = spawn(process.execPath, [
666
+ '-e',
667
+ xvfbCleanupMonitorScript,
668
+ String(parentPid),
669
+ String(xvfbPid)
670
+ ], {
671
+ detached: true,
672
+ stdio: 'ignore'
673
+ });
674
+ monitor.on('error', (error)=>{
675
+ debugXvfb(`Xvfb cleanup monitor failed: ${error.message}`);
676
+ });
677
+ instance.process.unref();
678
+ monitor.unref();
679
+ return monitor;
680
+ }
681
+ function createXvfbSignalCleanup(cleanup, source = process) {
682
+ return ()=>{
683
+ if (!hasActiveCliInterruptWaiter(source)) cleanup();
684
+ };
685
+ }
686
+ function checkXvfbInstalled() {
687
+ try {
688
+ execSync('which Xvfb', {
689
+ stdio: 'ignore'
690
+ });
691
+ return true;
692
+ } catch {
693
+ return false;
694
+ }
695
+ }
696
+ function findAvailableDisplay(startFrom = 99) {
697
+ for(let n = startFrom; n < startFrom + 100; n++)if (!existsSync(`/tmp/.X${n}-lock`)) return n;
698
+ throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
699
+ }
700
+ function startXvfb(options) {
701
+ const resolution = options?.resolution || '1920x1080x24';
702
+ const displayNum = options?.displayNumber ?? findAvailableDisplay();
703
+ const display = `:${displayNum}`;
704
+ return new Promise((resolve, reject)=>{
705
+ debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
706
+ const xvfbProcess = spawn('Xvfb', [
707
+ display,
708
+ '-screen',
709
+ '0',
710
+ resolution,
711
+ '-ac',
712
+ '-nolisten',
713
+ 'tcp'
714
+ ], {
715
+ stdio: 'ignore'
716
+ });
717
+ let settled = false;
718
+ xvfbProcess.on('error', (err)=>{
719
+ if (!settled) {
720
+ settled = true;
721
+ reject(new Error(`Failed to start Xvfb: ${err.message}`));
722
+ }
723
+ });
724
+ xvfbProcess.on('exit', (code)=>{
725
+ if (!settled) {
726
+ settled = true;
727
+ reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
728
+ }
729
+ });
730
+ const instance = {
731
+ process: xvfbProcess,
732
+ display,
733
+ stop () {
734
+ try {
735
+ xvfbProcess.kill('SIGTERM');
736
+ } catch {}
737
+ }
738
+ };
739
+ setTimeout(()=>{
740
+ if (!settled) {
741
+ settled = true;
742
+ debugXvfb(`Xvfb started on display ${display}`);
743
+ resolve(instance);
744
+ }
745
+ }, 500);
746
+ });
747
+ }
748
+ function needsXvfb(explicitOpt) {
749
+ if ('linux' !== process.platform) return false;
750
+ return true === explicitOpt;
751
+ }
752
+ function device_define_property(obj, key, value) {
753
+ if (key in obj) Object.defineProperty(obj, key, {
754
+ value: value,
755
+ enumerable: true,
756
+ configurable: true,
757
+ writable: true
758
+ });
759
+ else obj[key] = value;
760
+ return obj;
761
+ }
762
+ async function captureScreenshot(options) {
763
+ if ('linux' !== process.platform) return screenshot_desktop(options);
764
+ const filename = join(tmpdir(), `midscene-screenshot-${randomUUID()}.png`);
765
+ try {
766
+ await screenshot_desktop({
767
+ ...options,
768
+ filename
769
+ });
770
+ return readFileSync(filename);
771
+ } finally{
772
+ rmSync(filename, {
773
+ force: true
774
+ });
775
+ }
776
+ }
777
+ const SMOOTH_MOVE_STEPS_TAP = 8;
778
+ const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
779
+ const SMOOTH_MOVE_STEPS_DRAG = 20;
780
+ const SMOOTH_MOVE_DELAY_TAP = 8;
781
+ const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
782
+ const SMOOTH_MOVE_DELAY_DRAG = 10;
783
+ const MOUSE_MOVE_EFFECT_WAIT = 300;
784
+ const CLICK_SETTLE_DELAY = 50;
785
+ const CLICK_HOLD_DURATION = 100;
786
+ const CLICK_FOCUS_SETTLE_DELAY = 120;
787
+ const INPUT_FOCUS_DELAY = 300;
788
+ const INPUT_CLEAR_DELAY = 150;
789
+ const SCROLL_STEP_DELAY = 100;
790
+ const SCROLL_COMPLETE_DELAY = 500;
791
+ const EDGE_SCROLL_TOTAL_PX = 50000;
792
+ const EDGE_SCROLL_STEPS = 400;
793
+ const PHASED_PIXELS_PER_STEP = 30;
794
+ const PHASED_MIN_STEPS = 10;
795
+ const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
796
+ const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
797
+ const LIBNUT_FALLBACK_MAX_DETENTS = 200;
798
+ const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
799
+ const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
800
+ const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
801
+ const EDGE_SCROLL_SPEC = {
802
+ scrollToTop: {
803
+ direction: 'up',
804
+ key: 'home',
805
+ libnut: [
806
+ 0,
807
+ 1
808
+ ]
809
+ },
810
+ scrollToBottom: {
811
+ direction: 'down',
812
+ key: 'end',
813
+ libnut: [
814
+ 0,
815
+ -1
816
+ ]
817
+ },
818
+ scrollToLeft: {
819
+ direction: 'left',
820
+ key: 'home',
821
+ libnut: [
822
+ -1,
823
+ 0
824
+ ]
825
+ },
826
+ scrollToRight: {
827
+ direction: 'right',
828
+ key: 'end',
829
+ libnut: [
830
+ 1,
831
+ 0
832
+ ]
833
+ }
834
+ };
835
+ function escapePowershellSingleQuoted(value) {
836
+ return value.replace(/'/g, "''");
837
+ }
838
+ function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
839
+ return geometries.map((display)=>({
840
+ id: display.id,
841
+ name: display.name,
842
+ primary: display.primary
843
+ }));
844
+ }
845
+ let device_libnut = null;
846
+ let libnutLoadError = null;
847
+ async function getLibnut() {
848
+ if (device_libnut) return device_libnut;
849
+ if (libnutLoadError) throw libnutLoadError;
850
+ try {
851
+ const require = createRequire(import.meta.url);
852
+ const libnutModule = require('@computer-use/libnut/dist/import_libnut');
853
+ device_libnut = libnutModule.libnut;
854
+ if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
855
+ return device_libnut;
856
+ } catch (error) {
857
+ libnutLoadError = error;
858
+ throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
859
+ }
860
+ }
861
+ const debugDevice = getDebug('computer:device');
862
+ const warnDevice = getDebug('computer:device', {
863
+ console: true
864
+ });
865
+ const debugComputerInput = getDebug('computer:input', {
866
+ console: true
867
+ });
868
+ const WINDOWS_UIPI_DOC_URL = 'https://midscenejs.com/computer-getting-started#windows-clicks-have-no-effect-on-some-apps';
869
+ function resolvePackageRoot(helperName) {
870
+ const require = createRequire(import.meta.url);
871
+ let pkgRoot = null;
872
+ try {
873
+ pkgRoot = dirname(require.resolve('@midscene/computer/package.json'));
874
+ } catch {
875
+ const hereDir = dirname(fileURLToPath(import.meta.url));
876
+ for (const candidate of [
877
+ external_node_path_resolve(hereDir, '..'),
878
+ external_node_path_resolve(hereDir, '../..')
879
+ ])if (existsSync(external_node_path_resolve(candidate, 'package.json'))) {
880
+ pkgRoot = candidate;
881
+ break;
882
+ }
883
+ }
884
+ if (!pkgRoot) {
885
+ debugDevice(`${helperName}: cannot locate @midscene/computer package root`);
886
+ return null;
887
+ }
888
+ return pkgRoot;
889
+ }
890
+ let phasedScrollBinaryPath;
891
+ function getPhasedScrollBinary() {
892
+ if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
893
+ if ('darwin' !== process.platform) {
894
+ phasedScrollBinaryPath = null;
895
+ return null;
896
+ }
897
+ const pkgRoot = resolvePackageRoot('phased-scroll');
898
+ if (!pkgRoot) {
899
+ phasedScrollBinaryPath = null;
900
+ return null;
901
+ }
902
+ const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/phased-scroll');
903
+ if (!existsSync(binPath)) {
904
+ debugDevice('phased-scroll binary not found at', binPath);
905
+ phasedScrollBinaryPath = null;
906
+ return null;
907
+ }
908
+ try {
909
+ const st = statSync(binPath);
910
+ if ((73 & st.mode) === 0) {
911
+ chmodSync(binPath, 493);
912
+ debugDevice('phased-scroll: restored executable bit on', binPath);
913
+ }
914
+ } catch (err) {
915
+ debugDevice('phased-scroll: chmod self-heal failed', err);
916
+ }
917
+ phasedScrollBinaryPath = binPath;
918
+ return binPath;
919
+ }
920
+ let displayInfoBinaryPath;
921
+ function getDisplayInfoBinary() {
922
+ if (void 0 !== displayInfoBinaryPath) return displayInfoBinaryPath;
923
+ if ('darwin' !== process.platform) {
924
+ displayInfoBinaryPath = null;
925
+ return null;
926
+ }
927
+ const pkgRoot = resolvePackageRoot('display-info');
928
+ if (!pkgRoot) {
929
+ displayInfoBinaryPath = null;
930
+ return null;
931
+ }
932
+ const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/display-info');
933
+ if (!existsSync(binPath)) {
934
+ debugDevice('display-info binary not found at', binPath);
935
+ displayInfoBinaryPath = null;
936
+ return null;
937
+ }
938
+ displayInfoBinaryPath = binPath;
939
+ return binPath;
940
+ }
941
+ function device_isFiniteNumber(value) {
942
+ return 'number' == typeof value && Number.isFinite(value);
943
+ }
944
+ function isDarwinDisplayGeometry(value) {
945
+ if (!value || 'object' != typeof value) return false;
946
+ const candidate = value;
947
+ return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && device_isFiniteNumber(candidate.bounds.x) && device_isFiniteNumber(candidate.bounds.y) && device_isFiniteNumber(candidate.bounds.width) && device_isFiniteNumber(candidate.bounds.height);
948
+ }
949
+ function readDarwinDisplayGeometries() {
950
+ const bin = getDisplayInfoBinary();
951
+ if (!bin) return [];
952
+ try {
953
+ const output = execFileSync(bin, [], {
954
+ encoding: 'utf8',
955
+ stdio: [
956
+ 'ignore',
957
+ 'pipe',
958
+ 'pipe'
959
+ ]
960
+ });
961
+ const parsed = JSON.parse(output);
962
+ return Array.isArray(parsed.displays) ? parsed.displays.filter(isDarwinDisplayGeometry) : [];
963
+ } catch (error) {
964
+ debugDevice('display-info helper failed:', error);
965
+ return [];
966
+ }
967
+ }
968
+ function readDarwinFrontmostApplication() {
969
+ try {
970
+ const output = execFileSync("osascript", [
971
+ '-e',
972
+ 'tell application "System Events"\nset frontApp to first application process whose frontmost is true\nreturn (unix id of frontApp as string) & " " & (name of frontApp as string)\nend tell'
973
+ ]).toString().trim();
974
+ const [pidText, ...nameParts] = output.split('\t');
975
+ const pid = Number(pidText);
976
+ if (!Number.isInteger(pid) || pid <= 0) return;
977
+ return {
978
+ pid,
979
+ name: nameParts.join('\t')
980
+ };
981
+ } catch (error) {
982
+ debugDevice('Failed to read macOS frontmost application:', error);
983
+ return;
984
+ }
985
+ }
986
+ async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, current, holdDuration, reason) {
987
+ const drift = {
988
+ x: current.x - targetX,
989
+ y: current.y - targetY
990
+ };
991
+ debugComputerInput('tap mouse moved %o', {
992
+ reason,
993
+ target: {
994
+ x: targetX,
995
+ y: targetY
996
+ },
997
+ current,
998
+ drift
999
+ });
1000
+ await inputDriver.withMouseButton('left', async ()=>{
1001
+ debugComputerInput('tap mouse down %o', {
1002
+ reason
1003
+ });
1004
+ await inputDriver.delay(holdDuration);
1005
+ });
1006
+ debugComputerInput('tap mouse up %o', {
1007
+ reason
1008
+ });
1009
+ }
1010
+ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
1011
+ if (!displays.length) return;
1012
+ const screenIndex = void 0 === displayId || '' === displayId ? 0 : Number(displayId);
1013
+ if (!Number.isInteger(screenIndex) || screenIndex < 0) return void debugDevice('Invalid macOS display id for display geometry:', displayId);
1014
+ if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
1015
+ return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
1016
+ }
1017
+ function resolveDisplayGeometry(displayId, windowsDisplays) {
1018
+ if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
1019
+ if ('win32' === process.platform) {
1020
+ const geometry = resolveWindowsDisplayGeometryFromList(displayId, windowsDisplays ?? readWindowsDisplayGeometries());
1021
+ if (!geometry) throw new Error(displayId ? `Requested Windows display not found: ${displayId}` : 'No Windows displays were detected');
1022
+ return geometry;
1023
+ }
1024
+ }
1025
+ function mapDisplayLocalPointToGlobal(point, geometry) {
1026
+ if (!geometry) return point;
1027
+ return {
1028
+ x: point.x + geometry.bounds.x,
1029
+ y: point.y + geometry.bounds.y
1030
+ };
1031
+ }
1032
+ let phasedScrollExecWarned = false;
1033
+ function runPhasedScroll(direction, pixels, steps) {
1034
+ const bin = getPhasedScrollBinary();
1035
+ if (!bin) return false;
1036
+ try {
1037
+ const res = spawnSync(bin, [
1038
+ direction,
1039
+ String(Math.max(1, Math.round(pixels))),
1040
+ String(steps)
1041
+ ], {
1042
+ stdio: 'ignore'
1043
+ });
1044
+ if (0 === res.status) return true;
1045
+ if (!phasedScrollExecWarned) {
1046
+ phasedScrollExecWarned = true;
1047
+ const hint = null === res.status ? `signal ${res.signal ?? 'unknown'}; the binary may not be executable (npm tarball extraction can drop the +x bit) or may be blocked by quarantine. Try: chmod +x "${bin}"` : 'this usually means Accessibility permission has not been granted to the host process (System Settings → Privacy & Security → Accessibility)';
1048
+ console.warn(`[@midscene/computer] phased-scroll helper failed (exit=${res.status}, signal=${res.signal ?? 'none'}); falling back to keyboard/libnut. ${hint}`);
1049
+ }
1050
+ debugDevice('phased-scroll exited non-zero', res.status, res.signal, res.error);
1051
+ return false;
1052
+ } catch (err) {
1053
+ if (!phasedScrollExecWarned) {
1054
+ phasedScrollExecWarned = true;
1055
+ console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
1056
+ }
1057
+ debugDevice('phased-scroll spawn failed', err);
1058
+ return false;
1059
+ }
1060
+ }
1061
+ const KEY_NAME_MAP = {
1062
+ windows: 'win',
1063
+ win: 'win',
1064
+ ctrl: 'control',
1065
+ esc: 'escape',
1066
+ del: 'delete',
1067
+ ins: 'insert',
1068
+ pgup: 'pageup',
1069
+ pgdn: 'pagedown',
1070
+ arrowup: 'up',
1071
+ arrowdown: 'down',
1072
+ arrowleft: 'left',
1073
+ arrowright: 'right',
1074
+ volumedown: 'audio_vol_down',
1075
+ volumeup: 'audio_vol_up',
1076
+ mediavolumedown: 'audio_vol_down',
1077
+ mediavolumeup: 'audio_vol_up',
1078
+ mute: 'audio_mute',
1079
+ mediamute: 'audio_mute',
1080
+ mediaplay: 'audio_play',
1081
+ mediapause: 'audio_pause',
1082
+ mediaplaypause: 'audio_play',
1083
+ mediastop: 'audio_stop',
1084
+ medianexttrack: 'audio_next',
1085
+ mediaprevioustrack: 'audio_prev',
1086
+ medianext: 'audio_next',
1087
+ mediaprev: 'audio_prev'
1088
+ };
1089
+ const PRIMARY_KEY_MAP = {
1090
+ command: 'cmd',
1091
+ cmd: 'cmd',
1092
+ meta: 'meta',
1093
+ control: 'control',
1094
+ ctrl: 'control',
1095
+ shift: 'shift',
1096
+ alt: 'alt',
1097
+ option: 'alt'
1098
+ };
1099
+ function normalizeKeyName(key) {
1100
+ const lowerKey = key.toLowerCase();
1101
+ return KEY_NAME_MAP[lowerKey] || lowerKey;
1102
+ }
1103
+ function normalizePrimaryKey(key) {
1104
+ const lowerKey = key.toLowerCase();
1105
+ if (PRIMARY_KEY_MAP[lowerKey]) return PRIMARY_KEY_MAP[lowerKey];
1106
+ return KEY_NAME_MAP[lowerKey] || lowerKey;
1107
+ }
1108
+ class ComputerDevice {
1109
+ async moveGlobalPointer(point, context, smooth) {
1110
+ if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
1111
+ const target = {
1112
+ x: Math.round(point.x),
1113
+ y: Math.round(point.y)
1114
+ };
1115
+ if ('win32' === process.platform) {
1116
+ const actual = this.windowsPointerDriver.moveTo(target, {
1117
+ smoothSteps: smooth?.smoothSteps,
1118
+ smoothDelayMs: smooth?.smoothDelay
1119
+ });
1120
+ const drift = windowsPointerDrift(target, actual);
1121
+ if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`${context}: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
1122
+ await this.inputDriver.delay(CLICK_SETTLE_DELAY);
1123
+ return actual;
1124
+ }
1125
+ if (smooth) await this.inputDriver.smoothMoveMouse(target.x, target.y, smooth.smoothSteps, smooth.smoothDelay);
1126
+ else this.inputDriver.moveMouse(target.x, target.y);
1127
+ await this.inputDriver.delay(CLICK_SETTLE_DELAY);
1128
+ return this.inputDriver.getMousePos();
1129
+ }
1130
+ moveDisplayPointer(point, context, smooth) {
1131
+ return this.moveGlobalPointer(this.toGlobalPoint(point), context, smooth);
1132
+ }
1133
+ async performPointerDrag(from, to, duration) {
1134
+ const screenSize = await this.size();
1135
+ const boundedFrom = clampPointerPointToSize(from, screenSize);
1136
+ const boundedTo = clampPointerPointToSize(to, screenSize);
1137
+ await this.moveDisplayPointer(boundedFrom, 'Mouse did not reach the drag start target');
1138
+ await this.inputDriver.withMouseButton('left', async ()=>{
1139
+ await this.inputDriver.delay(100);
1140
+ await this.moveDisplayPointer(boundedTo, 'Mouse did not reach the drag end target', {
1141
+ smoothSteps: SMOOTH_MOVE_STEPS_DRAG,
1142
+ smoothDelay: void 0 === duration ? SMOOTH_MOVE_DELAY_DRAG : Math.max(0, Math.round(duration / SMOOTH_MOVE_STEPS_DRAG))
1143
+ });
1144
+ await this.inputDriver.delay(100);
1145
+ });
1146
+ }
1147
+ async focusKeyboardTarget(element, delayMs) {
1148
+ const [x, y] = element.center;
1149
+ if ('darwin' === process.platform) await this.inputPrimitives.pointer.tap({
1150
+ x,
1151
+ y
1152
+ });
1153
+ else {
1154
+ await this.moveDisplayPointer({
1155
+ x,
1156
+ y
1157
+ }, 'Mouse did not reach the keyboard focus target');
1158
+ this.inputDriver.mouseClick('left');
1159
+ }
1160
+ await this.inputDriver.delay(delayMs);
1161
+ }
1162
+ describe() {
1163
+ return this.description || 'Computer Device';
1164
+ }
1165
+ static async listDisplays() {
1166
+ try {
1167
+ if ('win32' === process.platform) return listWindowsDisplays();
1168
+ const displays = await screenshot_desktop.listDisplays();
1169
+ return displays.map((d)=>({
1170
+ id: String(d.id),
1171
+ name: d.name || `Display ${d.id}`,
1172
+ primary: d.primary || false
1173
+ }));
1174
+ } catch (error) {
1175
+ debugDevice(`Failed to list displays: ${error}`);
1176
+ throw new Error(`Failed to list displays: ${error}`);
1177
+ }
1178
+ }
1179
+ async connect() {
1180
+ debugDevice('Connecting to computer device');
1181
+ try {
1182
+ const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
1183
+ if (needsXvfb(headless)) {
1184
+ if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
1185
+ this.xvfbInstance = await startXvfb({
1186
+ resolution: this.options?.xvfbResolution
1187
+ });
1188
+ if (this.options?.keepXvfbAliveUntilProcessExit) scheduleXvfbStopAfterProcessExit(this.xvfbInstance);
1189
+ process.env.DISPLAY = this.xvfbInstance.display;
1190
+ debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
1191
+ if (!this.options?.keepXvfbAliveUntilProcessExit) {
1192
+ this.xvfbCleanup = ()=>{
1193
+ if (this.xvfbInstance) {
1194
+ this.xvfbInstance.stop();
1195
+ this.xvfbInstance = void 0;
1196
+ }
1197
+ };
1198
+ this.xvfbSignalCleanup = createXvfbSignalCleanup(()=>this.xvfbCleanup?.());
1199
+ process.on('exit', this.xvfbCleanup);
1200
+ process.on('SIGINT', this.xvfbSignalCleanup);
1201
+ process.on('SIGTERM', this.xvfbSignalCleanup);
1202
+ }
1203
+ }
1204
+ device_libnut = await getLibnut();
1205
+ const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
1206
+ this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
1207
+ const size = await this.size();
1208
+ const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
1209
+ const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
1210
+ this.description = `
1211
+ Type: Computer
1212
+ Platform: ${process.platform}
1213
+ Display: ${this.displayId || 'Primary'}
1214
+ Screen Size: ${size.width}x${size.height}
1215
+ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
1216
+ `;
1217
+ debugDevice('Computer device connected', this.description);
1218
+ await this.healthCheck(displays);
1219
+ } catch (error) {
1220
+ if (this.xvfbInstance) {
1221
+ if (!this.options?.keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
1222
+ this.xvfbInstance = void 0;
1223
+ }
1224
+ if (this.xvfbCleanup) {
1225
+ process.removeListener('exit', this.xvfbCleanup);
1226
+ this.xvfbCleanup = void 0;
1227
+ }
1228
+ if (this.xvfbSignalCleanup) {
1229
+ process.removeListener('SIGINT', this.xvfbSignalCleanup);
1230
+ process.removeListener('SIGTERM', this.xvfbSignalCleanup);
1231
+ this.xvfbSignalCleanup = void 0;
1232
+ }
1233
+ debugDevice(`Failed to connect: ${error}`);
1234
+ throw new Error(`Unable to connect to computer device: ${error}`);
1235
+ }
1236
+ }
1237
+ async healthCheck(displays) {
1238
+ console.log('[HealthCheck] Starting health check...');
1239
+ console.log("[HealthCheck] @midscene/computer v1.17.8-beta-20261110095715.0");
1240
+ console.log('[HealthCheck] Taking screenshot...');
1241
+ const screenshotTimeout = 15000;
1242
+ let timeoutId;
1243
+ const timeoutPromise = new Promise((_, reject)=>{
1244
+ timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
1245
+ });
1246
+ const base64 = await Promise.race([
1247
+ this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
1248
+ timeoutPromise
1249
+ ]);
1250
+ console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
1251
+ console.log('[HealthCheck] Verifying mouse control...');
1252
+ const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
1253
+ console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
1254
+ if ('win32' === process.platform) {
1255
+ if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
1256
+ const bounds = this.displayGeometry.bounds;
1257
+ const target = {
1258
+ x: Math.round(bounds.x + bounds.width / 2),
1259
+ y: Math.round(bounds.y + bounds.height / 2)
1260
+ };
1261
+ try {
1262
+ const actual = this.windowsPointerDriver.moveTo(target);
1263
+ const drift = windowsPointerDrift(target, actual);
1264
+ if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`Windows screenshot-space pointer verification: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
1265
+ } finally{
1266
+ this.windowsPointerDriver.moveTo(startPos);
1267
+ }
1268
+ console.log(`[HealthCheck] Mouse verified in screenshot-space display bounds (${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`);
1269
+ } else {
1270
+ const offsetX = Math.floor(40 * Math.random()) + 10;
1271
+ const offsetY = Math.floor(40 * Math.random()) + 10;
1272
+ const targetX = startPos.x + offsetX;
1273
+ const targetY = startPos.y + offsetY;
1274
+ console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
1275
+ try {
1276
+ this.inputDriver.moveMouse(targetX, targetY);
1277
+ await sleep(CLICK_SETTLE_DELAY);
1278
+ this.inputDriver.assertMousePosition(targetX, targetY, 'Mouse health check failed');
1279
+ } finally{
1280
+ this.inputDriver.moveMouse(startPos.x, startPos.y);
1281
+ console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
1282
+ }
1283
+ }
1284
+ if ('win32' === process.platform && !this.isRunningAsAdmin()) {
1285
+ const hint = [
1286
+ 'Heads-up: Midscene is not running as Administrator.',
1287
+ 'If clicks or key presses have no effect while the cursor still moves to the right position,',
1288
+ `see the Windows permission troubleshooting guide: ${WINDOWS_UIPI_DOC_URL}`
1289
+ ].join(' ');
1290
+ warnDevice(`[HealthCheck] ${hint}`);
1291
+ }
1292
+ console.log('[HealthCheck] Listing monitors...');
1293
+ if (displays.length > 0) {
1294
+ console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
1295
+ for (const display of displays){
1296
+ const primaryTag = display.primary ? ' (primary)' : '';
1297
+ console.log(`[HealthCheck] - id=${display.id}, name=${display.name}${primaryTag}`);
1298
+ }
1299
+ } else console.log('[HealthCheck] No monitors detected');
1300
+ console.log('[HealthCheck] Health check passed');
1301
+ }
1302
+ isRunningAsAdmin() {
1303
+ if ('win32' !== process.platform) return false;
1304
+ if (void 0 !== this.adminCheckCache) return this.adminCheckCache;
1305
+ try {
1306
+ execSync('net session', {
1307
+ stdio: 'pipe'
1308
+ });
1309
+ this.adminCheckCache = true;
1310
+ } catch {
1311
+ this.adminCheckCache = false;
1312
+ }
1313
+ return this.adminCheckCache;
1314
+ }
1315
+ async screenshotBase64() {
1316
+ if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
1317
+ debugDevice('Taking screenshot', {
1318
+ displayId: this.displayId
1319
+ });
1320
+ if ('win32' === process.platform) return this.screenshotViaPowershell();
1321
+ const options = {
1322
+ format: 'png'
1323
+ };
1324
+ if (void 0 !== this.displayId) if ('darwin' === process.platform) {
1325
+ const screenIndex = Number(this.displayId);
1326
+ if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
1327
+ } else options.screen = this.displayId;
1328
+ debugDevice('Screenshot options', options);
1329
+ const MAX_ATTEMPTS = 3;
1330
+ const RETRY_DELAY_MS = 300;
1331
+ let lastRawMessage = '';
1332
+ for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
1333
+ const buffer = await captureScreenshot(options);
1334
+ if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
1335
+ return createImgBase64ByFormat('png', buffer.toString('base64'));
1336
+ } catch (error) {
1337
+ lastRawMessage = error instanceof Error ? error.message : String(error);
1338
+ const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
1339
+ const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
1340
+ debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
1341
+ if (!willRetry) break;
1342
+ await sleep(RETRY_DELAY_MS);
1343
+ }
1344
+ if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
1345
+
1346
+ Please follow these steps:
1347
+ 1. Open System Settings > Privacy & Security > Screen Recording
1348
+ 2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
1349
+ 3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
1350
+
1351
+ Original error: ${lastRawMessage}`);
1352
+ throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
1353
+ }
1354
+ screenshotViaPowershell() {
1355
+ const deviceName = this.displayId ? String(this.displayId) : '';
1356
+ const selectScreen = deviceName ? `$dn = '${escapePowershellSingleQuoted(deviceName)}'
1357
+ $screen = [System.Windows.Forms.Screen]::AllScreens | Where-Object { $_.DeviceName -eq $dn } | Select-Object -First 1
1358
+ if (-not $screen) { throw "Requested display not found: $dn" }` : '$screen = [System.Windows.Forms.Screen]::PrimaryScreen';
1359
+ const script = `
1360
+ Add-Type -AssemblyName System.Windows.Forms, System.Drawing
1361
+ ${selectScreen}
1362
+ $b = $screen.Bounds
1363
+ $bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height)
1364
+ $g = [System.Drawing.Graphics]::FromImage($bmp)
1365
+ $g.CopyFromScreen($b.X, $b.Y, 0, 0, $bmp.Size)
1366
+ $ms = New-Object System.IO.MemoryStream
1367
+ $bmp.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png)
1368
+ [Console]::Out.Write([Convert]::ToBase64String($ms.ToArray()))
1369
+ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
1370
+ `.trim();
1371
+ let stdout;
1372
+ try {
1373
+ stdout = runWindowsPhysicalPixelPowershell(script);
1374
+ } catch (error) {
1375
+ const message = error instanceof Error ? error.message : String(error);
1376
+ throw new Error(`Failed to take screenshot on Windows: ${message}`);
1377
+ }
1378
+ const body = stdout.trim();
1379
+ if (!body) throw new Error('Failed to take screenshot on Windows: PowerShell returned no image data');
1380
+ return createImgBase64ByFormat('png', body);
1381
+ }
1382
+ async size() {
1383
+ if (this.displayGeometry) return {
1384
+ width: Math.round(this.displayGeometry.bounds.width),
1385
+ height: Math.round(this.displayGeometry.bounds.height)
1386
+ };
1387
+ try {
1388
+ const screenSize = this.inputDriver.getScreenSize();
1389
+ return {
1390
+ width: screenSize.width,
1391
+ height: screenSize.height
1392
+ };
1393
+ } catch (error) {
1394
+ debugDevice(`Failed to get screen size: ${error}`);
1395
+ throw new Error(`Failed to get screen size: ${error}`);
1396
+ }
1397
+ }
1398
+ toGlobalPoint(point) {
1399
+ return mapDisplayLocalPointToGlobal(point, this.displayGeometry);
1400
+ }
1401
+ async typeViaClipboard(text) {
1402
+ debugDevice('Using clipboard to input text', {
1403
+ textLength: text.length,
1404
+ preview: text.substring(0, 20)
1405
+ });
1406
+ const clipboardy = await import("clipboardy");
1407
+ const oldClipboard = await clipboardy.default.read().catch(()=>'');
1408
+ try {
1409
+ await clipboardy.default.write(text);
1410
+ await this.inputDriver.delay(50);
1411
+ if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
1412
+ 'command'
1413
+ ]);
1414
+ else {
1415
+ const modifier = 'darwin' === process.platform ? 'command' : 'control';
1416
+ await this.inputDriver.keyTapWithModifierDelay('v', [
1417
+ modifier
1418
+ ], this.options?.keyboardModifierDelay ?? 0);
1419
+ }
1420
+ await this.inputDriver.delay(100);
1421
+ } finally{
1422
+ if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
1423
+ debugDevice('Failed to restore clipboard content');
1424
+ });
1425
+ }
1426
+ }
1427
+ async smartTypeString(text, inputOptions) {
1428
+ if (shouldInputSequentially(inputOptions)) return void await this.typeStringWithDelay(text, inputOptions.keyboardTypeDelay ?? 0);
1429
+ await this.typeViaClipboard(text);
1430
+ }
1431
+ async typeStringWithDelay(text, keyboardTypeDelay) {
1432
+ await sendTextSequentially(text.replace(/\r\n?/g, '\n'), {
1433
+ sendCharacter: async (character)=>{
1434
+ const linuxShiftedKey = 'linux' === process.platform ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
1435
+ const pacedShiftedKey = !this.useAppleScript && (this.options?.keyboardModifierDelay ?? 0) > 0 ? resolveShiftedKey(character, this.options?.keyboardLayout) : void 0;
1436
+ if ('\n' === character) this.inputDriver.sendKey('return');
1437
+ else if ('\t' === character) this.inputDriver.sendKey('tab');
1438
+ else if (' ' === character) this.inputDriver.sendKey('space');
1439
+ else if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript(character);
1440
+ else if (void 0 !== pacedShiftedKey) await this.inputDriver.keyTapWithModifierDelay(pacedShiftedKey, [
1441
+ 'shift'
1442
+ ], this.options?.keyboardModifierDelay ?? 0);
1443
+ else if (void 0 !== linuxShiftedKey) this.inputDriver.keyTap(linuxShiftedKey, [
1444
+ 'shift'
1445
+ ]);
1446
+ else this.inputDriver.typeString(character);
1447
+ },
1448
+ wait: (delayMs)=>this.inputDriver.delay(delayMs)
1449
+ }, {
1450
+ delayMs: keyboardTypeDelay
1451
+ });
1452
+ }
1453
+ async selectAllAndDelete() {
1454
+ if (this.useAppleScript) {
1455
+ this.inputDriver.sendKeyViaAppleScript('a', [
1456
+ 'command'
1457
+ ]);
1458
+ await this.inputDriver.delay(50);
1459
+ this.inputDriver.sendKeyViaAppleScript('backspace', []);
1460
+ return;
1461
+ }
1462
+ const modifier = 'darwin' === process.platform ? 'command' : 'control';
1463
+ await this.inputDriver.keyTapWithModifierDelay('a', [
1464
+ modifier
1465
+ ], this.options?.keyboardModifierDelay ?? 0);
1466
+ await this.inputDriver.delay(50);
1467
+ this.inputDriver.keyTap('backspace');
1468
+ }
1469
+ async pressKeyboardShortcut(keyName) {
1470
+ const keys = keyName.split('+');
1471
+ const modifiers = keys.slice(0, -1).map(normalizeKeyName);
1472
+ const key = normalizePrimaryKey(keys[keys.length - 1]);
1473
+ debugDevice('KeyboardPress', {
1474
+ original: keyName,
1475
+ key,
1476
+ modifiers,
1477
+ driver: this.useAppleScript ? "applescript" : 'libnut'
1478
+ });
1479
+ if (!this.useAppleScript && modifiers.length > 0 && (this.options?.keyboardModifierDelay ?? 0) > 0) return void await this.inputDriver.keyTapWithModifierDelay(key, modifiers, this.options?.keyboardModifierDelay ?? 0);
1480
+ this.inputDriver.sendKey(key, modifiers);
1481
+ }
1482
+ resolveUntargetedScrollPoint(screenSize) {
1483
+ if ('win32' === process.platform) {
1484
+ const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
1485
+ if (activeWindowRect) {
1486
+ const activeWindowCenter = {
1487
+ x: activeWindowRect.x + activeWindowRect.width / 2,
1488
+ y: activeWindowRect.y + activeWindowRect.height / 2
1489
+ };
1490
+ const bounds = this.displayGeometry?.bounds;
1491
+ if (!bounds || activeWindowCenter.x >= bounds.x && activeWindowCenter.x < bounds.x + bounds.width && activeWindowCenter.y >= bounds.y && activeWindowCenter.y < bounds.y + bounds.height) return activeWindowCenter;
1492
+ }
1493
+ }
1494
+ return this.toGlobalPoint({
1495
+ x: screenSize.width / 2,
1496
+ y: screenSize.height / 2
1497
+ });
1498
+ }
1499
+ async moveMouseToScrollTarget(param) {
1500
+ if (param.locate) {
1501
+ const element = param.locate;
1502
+ const [x, y] = element.center;
1503
+ await this.moveDisplayPointer({
1504
+ x,
1505
+ y
1506
+ }, 'Mouse did not reach the scroll target');
1507
+ return;
1508
+ }
1509
+ const screenSize = await this.size();
1510
+ if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
1511
+ const point = this.resolveUntargetedScrollPoint(screenSize);
1512
+ await this.moveGlobalPointer(point, 'Mouse did not reach the scroll viewport');
1513
+ await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
1514
+ return screenSize;
1515
+ }
1516
+ async performScroll(param) {
1517
+ let screenSize = await this.moveMouseToScrollTarget(param);
1518
+ const scrollType = param?.scrollType;
1519
+ const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
1520
+ if (edgeSpec) {
1521
+ if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
1522
+ if (this.useAppleScript) {
1523
+ this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
1524
+ await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
1525
+ return;
1526
+ }
1527
+ const [ux, uy] = edgeSpec.libnut;
1528
+ await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
1529
+ return;
1530
+ }
1531
+ if ('singleAction' === scrollType || !scrollType) {
1532
+ const direction = param?.direction || 'down';
1533
+ const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
1534
+ const isHorizontal = 'left' === direction || 'right' === direction;
1535
+ let distance = param?.distance ?? void 0;
1536
+ if (!distance) {
1537
+ screenSize ??= await this.size();
1538
+ const base = isHorizontal ? screenSize.width : screenSize.height;
1539
+ distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
1540
+ }
1541
+ if (isKnownDirection) {
1542
+ const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
1543
+ if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
1544
+ }
1545
+ if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
1546
+ if (!screenSize) screenSize = await this.size();
1547
+ const pages = Math.max(1, Math.round(distance / screenSize.height));
1548
+ const key = 'up' === direction ? 'pageup' : 'pagedown';
1549
+ for(let i = 0; i < pages; i++){
1550
+ this.inputDriver.sendKeyViaAppleScript(key);
1551
+ await this.inputDriver.delay(SCROLL_STEP_DELAY);
1552
+ }
1553
+ await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
1554
+ return;
1555
+ }
1556
+ const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
1557
+ const directionUnit = {
1558
+ up: [
1559
+ 0,
1560
+ 1
1561
+ ],
1562
+ down: [
1563
+ 0,
1564
+ -1
1565
+ ],
1566
+ left: [
1567
+ -1,
1568
+ 0
1569
+ ],
1570
+ right: [
1571
+ 1,
1572
+ 0
1573
+ ]
1574
+ };
1575
+ const [ux, uy] = directionUnit[direction] || [
1576
+ 0,
1577
+ -1
1578
+ ];
1579
+ await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
1580
+ await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
1581
+ return;
1582
+ }
1583
+ throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
1584
+ }
1585
+ actionSpace() {
1586
+ const defaultActions = [
1587
+ ...defineActionsFromInputPrimitives(this.inputPrimitives, {
1588
+ size: ()=>this.size()
1589
+ })
1590
+ ];
1591
+ const platformActions = Object.values(createPlatformActions());
1592
+ const customActions = this.options?.customActions || [];
1593
+ return [
1594
+ ...defaultActions,
1595
+ ...platformActions,
1596
+ ...customActions
1597
+ ];
1598
+ }
1599
+ async destroy() {
1600
+ if (this.destroyed) return;
1601
+ this.destroyed = true;
1602
+ this.inputDriver.destroy();
1603
+ const keepXvfbAliveUntilProcessExit = this.options?.keepXvfbAliveUntilProcessExit === true;
1604
+ if (this.xvfbInstance) {
1605
+ if (!keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
1606
+ this.xvfbInstance = void 0;
1607
+ }
1608
+ if (this.xvfbCleanup && !keepXvfbAliveUntilProcessExit) {
1609
+ process.removeListener('exit', this.xvfbCleanup);
1610
+ this.xvfbCleanup = void 0;
1611
+ }
1612
+ if (this.xvfbSignalCleanup) {
1613
+ process.removeListener('SIGINT', this.xvfbSignalCleanup);
1614
+ process.removeListener('SIGTERM', this.xvfbSignalCleanup);
1615
+ this.xvfbSignalCleanup = void 0;
1616
+ }
1617
+ debugDevice('Computer device destroyed');
1618
+ }
1619
+ async url() {
1620
+ return '';
1621
+ }
1622
+ constructor(options){
1623
+ device_define_property(this, "interfaceType", 'computer');
1624
+ device_define_property(this, "options", void 0);
1625
+ device_define_property(this, "displayId", void 0);
1626
+ device_define_property(this, "displayGeometry", void 0);
1627
+ device_define_property(this, "description", void 0);
1628
+ device_define_property(this, "destroyed", false);
1629
+ device_define_property(this, "xvfbInstance", void 0);
1630
+ device_define_property(this, "xvfbCleanup", void 0);
1631
+ device_define_property(this, "xvfbSignalCleanup", void 0);
1632
+ device_define_property(this, "inputDriver", new ComputerInputDriver({
1633
+ getLibnut: ()=>device_libnut,
1634
+ useAppleScript: ()=>this.useAppleScript,
1635
+ sendKeyViaAppleScript: sendKeyViaAppleScript,
1636
+ runPhasedScroll: runPhasedScroll,
1637
+ debug: (message)=>debugDevice(message)
1638
+ }));
1639
+ device_define_property(this, "windowsPointerDriver", new WindowsPointerDriver({
1640
+ runPhysicalPixelPowershell: runWindowsPhysicalPixelPowershell
1641
+ }));
1642
+ device_define_property(this, "useAppleScript", void 0);
1643
+ device_define_property(this, "adminCheckCache", void 0);
1644
+ device_define_property(this, "uri", void 0);
1645
+ device_define_property(this, "inputPrimitives", {
1646
+ pointer: {
1647
+ tap: async ({ x, y }, opts)=>{
1648
+ const target = this.toGlobalPoint({
1649
+ x,
1650
+ y
1651
+ });
1652
+ const targetX = Math.round(target.x);
1653
+ const targetY = Math.round(target.y);
1654
+ const holdDuration = Math.max(0, Math.round(opts?.duration ?? CLICK_HOLD_DURATION));
1655
+ debugComputerInput('tap start %o', {
1656
+ local: {
1657
+ x,
1658
+ y
1659
+ },
1660
+ global: {
1661
+ x: targetX,
1662
+ y: targetY
1663
+ },
1664
+ holdDuration,
1665
+ displayId: this.displayId,
1666
+ displayGeometry: this.displayGeometry
1667
+ });
1668
+ const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
1669
+ const current = await this.moveGlobalPointer({
1670
+ x: targetX,
1671
+ y: targetY
1672
+ }, 'Mouse did not reach the tap target', {
1673
+ smoothSteps: SMOOTH_MOVE_STEPS_TAP,
1674
+ smoothDelay: SMOOTH_MOVE_DELAY_TAP
1675
+ });
1676
+ await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, current, holdDuration, 'primary');
1677
+ if (frontmostBefore && 'darwin' === process.platform) {
1678
+ await sleep(CLICK_FOCUS_SETTLE_DELAY);
1679
+ const frontmostAfter = readDarwinFrontmostApplication();
1680
+ const focusChanged = !!frontmostAfter && frontmostAfter.pid !== frontmostBefore.pid;
1681
+ debugComputerInput('tap focus check %o', {
1682
+ before: frontmostBefore,
1683
+ after: frontmostAfter,
1684
+ focusChanged
1685
+ });
1686
+ if (focusChanged) {
1687
+ const followUpCurrent = await this.moveGlobalPointer({
1688
+ x: targetX,
1689
+ y: targetY
1690
+ }, 'Mouse did not reach the focus follow-up target');
1691
+ await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, followUpCurrent, holdDuration, 'focus-follow-up');
1692
+ }
1693
+ }
1694
+ },
1695
+ doubleClick: async ({ x, y })=>{
1696
+ await this.moveDisplayPointer({
1697
+ x,
1698
+ y
1699
+ }, 'Mouse did not reach the double-click target');
1700
+ this.inputDriver.mouseClick('left', true);
1701
+ },
1702
+ rightClick: async ({ x, y })=>{
1703
+ await this.moveDisplayPointer({
1704
+ x,
1705
+ y
1706
+ }, 'Mouse did not reach the right-click target');
1707
+ this.inputDriver.mouseClick('right');
1708
+ },
1709
+ middleClick: async ({ x, y })=>{
1710
+ await this.moveDisplayPointer({
1711
+ x,
1712
+ y
1713
+ }, 'Mouse did not reach the middle-click target');
1714
+ this.inputDriver.mouseClick('middle');
1715
+ },
1716
+ hover: async ({ x, y })=>{
1717
+ await this.moveDisplayPointer({
1718
+ x,
1719
+ y
1720
+ }, 'Mouse did not reach the hover target', {
1721
+ smoothSteps: SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1722
+ smoothDelay: SMOOTH_MOVE_DELAY_MOUSE_MOVE
1723
+ });
1724
+ await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
1725
+ },
1726
+ dragAndDrop: async (from, to)=>{
1727
+ await this.performPointerDrag(from, to);
1728
+ },
1729
+ swipe: async (from, to, opts)=>{
1730
+ const repeatCount = opts?.repeat ?? 1;
1731
+ for(let index = 0; index < repeatCount; index++)await this.performPointerDrag(from, to, opts?.duration);
1732
+ }
1733
+ },
1734
+ keyboard: {
1735
+ typeText: async (value, opts)=>{
1736
+ const resolvedInputOptions = resolveTextInputOptions(opts, this.options);
1737
+ const element = opts?.target;
1738
+ if (element) {
1739
+ await this.focusKeyboardTarget(element, INPUT_FOCUS_DELAY);
1740
+ if (opts?.replace !== false) {
1741
+ await this.selectAllAndDelete();
1742
+ await this.inputDriver.delay(INPUT_CLEAR_DELAY);
1743
+ }
1744
+ }
1745
+ await this.smartTypeString(value, resolvedInputOptions);
1746
+ },
1747
+ keyboardPress: async (keyName, opts)=>{
1748
+ const target = opts?.target;
1749
+ if (target) await this.focusKeyboardTarget(target, 50);
1750
+ await this.pressKeyboardShortcut(keyName);
1751
+ },
1752
+ clearInput: async (target)=>{
1753
+ if (target) await this.focusKeyboardTarget(target, 100);
1754
+ await this.selectAllAndDelete();
1755
+ await this.inputDriver.delay(50);
1756
+ }
1757
+ },
1758
+ scroll: {
1759
+ scroll: async (param)=>{
1760
+ await this.performScroll(param);
1761
+ }
1762
+ }
1763
+ });
1764
+ if (options?.keyboardModifierDelay !== void 0 && (!Number.isFinite(options.keyboardModifierDelay) || options.keyboardModifierDelay < 0)) throw new Error('keyboardModifierDelay must be a finite non-negative number');
1765
+ if (options?.keyboardLayout !== void 0 && 'en-US' !== options.keyboardLayout) throw new Error('keyboardLayout must be "en-US" when specified');
1766
+ this.options = options;
1767
+ this.displayId = options?.displayId;
1768
+ this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
1769
+ }
1770
+ }
1771
+ function createPlatformActions() {
1772
+ return {
1773
+ ListDisplays: defineAction({
1774
+ name: 'ListDisplays',
1775
+ supportedBySystemOne: true,
1776
+ description: 'List all available displays/monitors',
1777
+ call: async ()=>await ComputerDevice.listDisplays()
1778
+ })
1779
+ };
1780
+ }
1781
+ function normalizeRdpHost(host) {
1782
+ const trimmed = host.trim();
1783
+ if (trimmed.length >= 2 && trimmed.startsWith('[') && trimmed.endsWith(']') && trimmed.includes(':')) return trimmed.slice(1, -1);
1784
+ return trimmed;
1785
+ }
1786
+ function formatRdpHost(host) {
1787
+ const normalizedHost = normalizeRdpHost(host);
1788
+ return normalizedHost.includes(':') ? `[${normalizedHost}]` : normalizedHost;
1789
+ }
1790
+ function formatRdpServerAddress(host, port) {
1791
+ return `${formatRdpHost(host)}:${port}`;
1792
+ }
1793
+ function normalizeRdpConnectionConfig(config) {
1794
+ return {
1795
+ ...config,
1796
+ host: normalizeRdpHost(config.host),
1797
+ ...config.localAddress ? {
1798
+ localAddress: config.localAddress.trim()
1799
+ } : {}
1800
+ };
1801
+ }
1802
+ const platformBinaryMap = {
1803
+ darwin: {
1804
+ directory: 'darwin',
1805
+ fileName: 'rdp-helper'
1806
+ },
1807
+ linux: {
1808
+ directory: 'linux',
1809
+ fileName: 'rdp-helper'
1810
+ },
1811
+ win32: {
1812
+ directory: 'win32',
1813
+ fileName: 'rdp-helper.exe'
1814
+ }
1815
+ };
1816
+ function getPlatformBinary(platform) {
1817
+ if (platform in platformBinaryMap) return platformBinaryMap[platform];
1818
+ }
1819
+ function currentDirname() {
1820
+ if ("u" > typeof __dirname) return __dirname;
1821
+ return dirname(fileURLToPath(import.meta.url));
1822
+ }
1823
+ function getRdpHelperBinaryPath() {
1824
+ const platformBinary = getPlatformBinary(process.platform);
1825
+ if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
1826
+ const hereDir = currentDirname();
1827
+ const candidateRoots = [
1828
+ external_node_path_resolve(hereDir, '../..'),
1829
+ external_node_path_resolve(hereDir, '../../..')
1830
+ ];
1831
+ for (const root of candidateRoots){
1832
+ const binaryPath = external_node_path_resolve(root, 'bin', platformBinary.directory, platformBinary.fileName);
1833
+ if (existsSync(binaryPath)) return binaryPath;
1834
+ }
1835
+ throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
1836
+ }
1837
+ function backend_client_define_property(obj, key, value) {
1838
+ if (key in obj) Object.defineProperty(obj, key, {
1839
+ value: value,
1840
+ enumerable: true,
1841
+ configurable: true,
1842
+ writable: true
1843
+ });
1844
+ else obj[key] = value;
1845
+ return obj;
1846
+ }
1847
+ const debug = getDebug('rdp:backend');
1848
+ const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
1849
+ const HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS = 50;
1850
+ const MAX_STDERR_CHARS = 16384;
1851
+ const notImplementedError = ()=>new Error('RDP backend transport is not implemented yet. Inject a custom backend into RDPDevice or implement a real helper transport.');
1852
+ class UnsupportedRDPBackendClient {
1853
+ async connect(_config) {
1854
+ debug('connect called on unsupported backend');
1855
+ throw notImplementedError();
1856
+ }
1857
+ async disconnect() {
1858
+ debug('disconnect called on unsupported backend');
1859
+ throw notImplementedError();
1860
+ }
1861
+ async screenshotBase64() {
1862
+ debug('screenshotBase64 called on unsupported backend');
1863
+ throw notImplementedError();
1864
+ }
1865
+ async size() {
1866
+ debug('size called on unsupported backend');
1867
+ throw notImplementedError();
1868
+ }
1869
+ async mouseMove(_x, _y) {
1870
+ debug('mouseMove called on unsupported backend');
1871
+ throw notImplementedError();
1872
+ }
1873
+ async mouseButton(_button, _action) {
1874
+ debug('mouseButton called on unsupported backend');
1875
+ throw notImplementedError();
1876
+ }
1877
+ async wheel(_direction, _amount, _x, _y) {
1878
+ debug('wheel called on unsupported backend');
1879
+ throw notImplementedError();
1880
+ }
1881
+ async keyPress(_keyName) {
1882
+ debug('keyPress called on unsupported backend');
1883
+ throw notImplementedError();
1884
+ }
1885
+ async typeText(_text) {
1886
+ debug('typeText called on unsupported backend');
1887
+ throw notImplementedError();
1888
+ }
1889
+ async clearInput() {
1890
+ debug('clearInput called on unsupported backend');
1891
+ throw notImplementedError();
1892
+ }
1893
+ }
1894
+ class HelperProcessRDPBackendClient {
1895
+ async connect(config) {
1896
+ this.fatalHelperError = void 0;
1897
+ const response = await this.send({
1898
+ type: 'connect',
1899
+ config: normalizeRdpConnectionConfig(config)
1900
+ });
1901
+ if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
1902
+ this.connected = true;
1903
+ this.fatalHelperError = void 0;
1904
+ return response.info;
1905
+ }
1906
+ async disconnect() {
1907
+ const child = this.child;
1908
+ if (!child) return;
1909
+ let disconnectError;
1910
+ if (this.connected && null === child.exitCode) try {
1911
+ const response = await this.send({
1912
+ type: 'disconnect'
1913
+ });
1914
+ this.expectOk(response, 'disconnect');
1915
+ } catch (error) {
1916
+ disconnectError = error instanceof Error ? error : new Error(String(error));
1917
+ }
1918
+ this.connected = false;
1919
+ this.fatalHelperError = void 0;
1920
+ await this.shutdownHelper();
1921
+ if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
1922
+ }
1923
+ async screenshotBase64() {
1924
+ const response = await this.send({
1925
+ type: 'screenshot'
1926
+ });
1927
+ if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
1928
+ return response.base64;
1929
+ }
1930
+ async size() {
1931
+ const response = await this.send({
1932
+ type: 'size'
1933
+ });
1934
+ if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
1935
+ return response.size;
1936
+ }
1937
+ async mouseMove(x, y) {
1938
+ const response = await this.send({
1939
+ type: 'mouseMove',
1940
+ x,
1941
+ y
1942
+ });
1943
+ this.expectOk(response, 'mouseMove');
1944
+ }
1945
+ async mouseButton(button, action) {
1946
+ const response = await this.send({
1947
+ type: 'mouseButton',
1948
+ button,
1949
+ action
1950
+ });
1951
+ this.expectOk(response, 'mouseButton');
1952
+ }
1953
+ async wheel(direction, amount, x, y) {
1954
+ const response = await this.send({
1955
+ type: 'wheel',
1956
+ direction,
1957
+ amount,
1958
+ x,
1959
+ y
1960
+ });
1961
+ this.expectOk(response, 'wheel');
1962
+ }
1963
+ async keyPress(keyName) {
1964
+ const response = await this.send({
1965
+ type: 'keyPress',
1966
+ keyName
1967
+ });
1968
+ this.expectOk(response, 'keyPress');
1969
+ }
1970
+ async typeText(text) {
1971
+ const response = await this.send({
1972
+ type: 'typeText',
1973
+ text
1974
+ });
1975
+ this.expectOk(response, 'typeText');
1976
+ }
1977
+ async clearInput() {
1978
+ const response = await this.send({
1979
+ type: 'clearInput'
1980
+ });
1981
+ this.expectOk(response, 'clearInput');
1982
+ }
1983
+ ensureHelperStarted() {
1984
+ if (this.child && null === this.child.exitCode) return this.child;
1985
+ const helperPath = this.resolveHelperPath();
1986
+ debug('starting rdp helper', {
1987
+ helperPath
1988
+ });
1989
+ const child = this.spawnFn(helperPath, [], {
1990
+ stdio: [
1991
+ 'pipe',
1992
+ 'pipe',
1993
+ 'pipe'
1994
+ ]
1995
+ });
1996
+ child.stdout.setEncoding('utf8');
1997
+ child.stderr.setEncoding('utf8');
1998
+ this.child = child;
1999
+ const diagnostics = {
2000
+ path: helperPath,
2001
+ pid: child.pid,
2002
+ stderrBuffer: ''
2003
+ };
2004
+ this.helperDiagnostics.set(child, diagnostics);
2005
+ debug('started rdp helper', {
2006
+ helperPath,
2007
+ pid: child.pid
2008
+ });
2009
+ this.stdoutReader = createInterface({
2010
+ input: child.stdout,
2011
+ crlfDelay: 1 / 0
2012
+ });
2013
+ this.stdoutReader.on('line', (line)=>{
2014
+ this.handleStdoutLine(child, line, diagnostics);
2015
+ });
2016
+ child.stderr.on('data', (chunk)=>{
2017
+ this.captureStderrChunk(chunk, diagnostics);
2018
+ });
2019
+ child.stdin.on('error', (error)=>{
2020
+ this.handleHelperStreamError(child, 'stdin', error, diagnostics);
2021
+ });
2022
+ child.stdout.on('error', (error)=>{
2023
+ this.handleHelperStreamError(child, 'stdout', error, diagnostics);
2024
+ });
2025
+ child.stderr.on('error', (error)=>{
2026
+ this.handleHelperStreamError(child, 'stderr', error, diagnostics);
2027
+ });
2028
+ child.on('exit', (code, signal)=>{
2029
+ diagnostics.exit = {
2030
+ code,
2031
+ signal
2032
+ };
2033
+ debug('rdp helper exited', {
2034
+ helperPath: diagnostics.path,
2035
+ pid: diagnostics.pid,
2036
+ code,
2037
+ signal
2038
+ });
2039
+ const helperError = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`, void 0, diagnostics);
2040
+ this.rejectPendingForChild(child, helperError);
2041
+ if (this.child === child) {
2042
+ this.connected = false;
2043
+ this.fatalHelperError = helperError;
2044
+ this.disposeReaders();
2045
+ this.child = void 0;
2046
+ }
2047
+ });
2048
+ child.on('error', (error)=>{
2049
+ const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`, void 0, diagnostics);
2050
+ this.rejectPendingForChild(child, helperError);
2051
+ if (this.child === child) {
2052
+ this.connected = false;
2053
+ this.fatalHelperError = helperError;
2054
+ this.disposeReaders();
2055
+ this.child = void 0;
2056
+ }
2057
+ });
2058
+ return child;
2059
+ }
2060
+ handleHelperStreamError(child, streamName, error, diagnostics) {
2061
+ const nodeError = error;
2062
+ const helperError = this.createHelperError(`RDP helper ${streamName} stream error: ${error.message}`, nodeError.code, diagnostics);
2063
+ this.rejectPendingForChild(child, helperError);
2064
+ if (this.child !== child) return;
2065
+ this.connected = false;
2066
+ this.fatalHelperError = helperError;
2067
+ if (null === child.exitCode) child.kill('SIGTERM');
2068
+ this.child = void 0;
2069
+ }
2070
+ handleStdoutLine(child, line, diagnostics) {
2071
+ if (!line.trim()) return;
2072
+ let parsed;
2073
+ try {
2074
+ parsed = JSON.parse(line);
2075
+ } catch (error) {
2076
+ const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`, void 0, diagnostics);
2077
+ this.rejectPendingForChild(child, protocolError);
2078
+ if (this.child === child) this.shutdownHelper(protocolError);
2079
+ return;
2080
+ }
2081
+ const pending = this.pending.get(parsed.id);
2082
+ if (!pending) return void debug('dropping response for unknown request id', parsed);
2083
+ this.pending.delete(parsed.id);
2084
+ if (parsed.ok) return void pending.resolve(parsed.payload);
2085
+ pending.reject(this.createHelperError(parsed.error.message, parsed.error.code, diagnostics));
2086
+ }
2087
+ captureStderrChunk(chunk, diagnostics) {
2088
+ const text = chunk.toString();
2089
+ if (!text.trim()) return;
2090
+ debug('rdp helper stderr', {
2091
+ helperPath: diagnostics.path,
2092
+ pid: diagnostics.pid,
2093
+ stderr: text.trim()
2094
+ });
2095
+ diagnostics.stderrBuffer += text;
2096
+ if (diagnostics.stderrBuffer.length > MAX_STDERR_CHARS) diagnostics.stderrBuffer = diagnostics.stderrBuffer.slice(-MAX_STDERR_CHARS);
2097
+ }
2098
+ async send(payload) {
2099
+ if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
2100
+ const child = this.ensureHelperStarted();
2101
+ if (null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
2102
+ const id = `req-${++this.nextRequestId}`;
2103
+ const diagnostics = this.helperDiagnostics.get(child);
2104
+ const request = {
2105
+ id,
2106
+ payload
2107
+ };
2108
+ return new Promise((resolve, reject)=>{
2109
+ this.pending.set(id, {
2110
+ resolve,
2111
+ reject,
2112
+ child
2113
+ });
2114
+ child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
2115
+ if (!error) return;
2116
+ this.pending.delete(id);
2117
+ const nodeError = error;
2118
+ const timer = setTimeout(()=>{
2119
+ reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`, nodeError.code, diagnostics));
2120
+ }, HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS);
2121
+ timer.unref?.();
2122
+ });
2123
+ });
2124
+ }
2125
+ expectOk(response, actionName) {
2126
+ if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
2127
+ }
2128
+ rejectPending(error) {
2129
+ for (const { reject } of this.pending.values())reject(error);
2130
+ this.pending.clear();
2131
+ }
2132
+ rejectPendingForChild(child, error) {
2133
+ for (const [id, pending] of this.pending)if (pending.child === child) {
2134
+ pending.reject(error);
2135
+ this.pending.delete(id);
2136
+ }
2137
+ }
2138
+ createHelperError(message, code, diagnostics) {
2139
+ const diagnosticParts = [
2140
+ diagnostics?.path ? `path=${diagnostics.path}` : void 0,
2141
+ 'number' == typeof diagnostics?.pid ? `pid=${diagnostics.pid}` : void 0,
2142
+ diagnostics?.exit ? `exitCode=${diagnostics.exit.code}, signal=${diagnostics.exit.signal}` : void 0
2143
+ ].filter(Boolean);
2144
+ const diagnosticsSuffix = diagnosticParts.length > 0 ? `\nHelper diagnostics: ${diagnosticParts.join(', ')}` : '';
2145
+ const stderrSummary = diagnostics?.stderrBuffer.trim();
2146
+ const stderrSuffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
2147
+ const error = new Error(`${message}${diagnosticsSuffix}${stderrSuffix}`);
2148
+ if (code) error.name = code;
2149
+ return error;
2150
+ }
2151
+ disposeReaders() {
2152
+ this.stdoutReader?.close();
2153
+ this.stdoutReader = void 0;
2154
+ }
2155
+ async shutdownHelper(rootError) {
2156
+ const child = this.child;
2157
+ this.child = void 0;
2158
+ this.disposeReaders();
2159
+ if (!child) return;
2160
+ this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
2161
+ if (null !== child.exitCode) return;
2162
+ child.stdin.end();
2163
+ const exited = Promise.race([
2164
+ once(child, 'exit'),
2165
+ new Promise((resolve)=>{
2166
+ setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
2167
+ })
2168
+ ]);
2169
+ const result = await exited;
2170
+ if ('timeout' !== result) return;
2171
+ child.kill('SIGTERM');
2172
+ const terminated = Promise.race([
2173
+ once(child, 'exit'),
2174
+ new Promise((resolve)=>{
2175
+ setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
2176
+ })
2177
+ ]);
2178
+ const terminateResult = await terminated;
2179
+ if ('timeout' !== terminateResult) return;
2180
+ child.kill('SIGKILL');
2181
+ await once(child, 'exit');
2182
+ }
2183
+ constructor(options){
2184
+ backend_client_define_property(this, "spawnFn", void 0);
2185
+ backend_client_define_property(this, "resolveHelperPath", void 0);
2186
+ backend_client_define_property(this, "child", void 0);
2187
+ backend_client_define_property(this, "stdoutReader", void 0);
2188
+ backend_client_define_property(this, "pending", new Map());
2189
+ backend_client_define_property(this, "helperDiagnostics", new WeakMap());
2190
+ backend_client_define_property(this, "nextRequestId", 0);
2191
+ backend_client_define_property(this, "connected", false);
2192
+ backend_client_define_property(this, "fatalHelperError", void 0);
2193
+ this.spawnFn = options?.spawnFn || spawn;
2194
+ const overridePath = options?.helperPath;
2195
+ this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
2196
+ }
2197
+ }
2198
+ function createDefaultRDPBackendClient() {
2199
+ return new HelperProcessRDPBackendClient();
2200
+ }
2201
+ function rdp_device_define_property(obj, key, value) {
2202
+ if (key in obj) Object.defineProperty(obj, key, {
2203
+ value: value,
2204
+ enumerable: true,
2205
+ configurable: true,
2206
+ writable: true
2207
+ });
2208
+ else obj[key] = value;
2209
+ return obj;
2210
+ }
2211
+ const device_debug = getDebug('rdp:device');
2212
+ const device_SMOOTH_MOVE_STEPS_TAP = 8;
2213
+ const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
2214
+ const device_SMOOTH_MOVE_STEPS_DRAG = 12;
2215
+ const device_SMOOTH_MOVE_DELAY_TAP = 8;
2216
+ const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
2217
+ const device_SMOOTH_MOVE_DELAY_DRAG = 10;
2218
+ const device_MOUSE_MOVE_EFFECT_WAIT = 300;
2219
+ const device_CLICK_HOLD_DURATION = 50;
2220
+ const DRAG_HOLD_DURATION = 100;
2221
+ const device_INPUT_FOCUS_DELAY = 300;
2222
+ const device_INPUT_CLEAR_DELAY = 150;
2223
+ const device_SCROLL_STEP_DELAY = 100;
2224
+ const device_SCROLL_COMPLETE_DELAY = 500;
2225
+ const DEFAULT_SCROLL_DISTANCE = 480;
2226
+ const device_DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
2227
+ const device_EDGE_SCROLL_STEPS = 10;
2228
+ const DEFAULT_SCROLL_STEP_AMOUNT = 120;
2229
+ class RDPDevice {
2230
+ async typeText(value, inputOptions) {
2231
+ if (!shouldInputSequentially(inputOptions)) return void await this.backend.typeText(value);
2232
+ await sendTextSequentially(value, {
2233
+ sendCharacter: (character)=>this.backend.typeText(character),
2234
+ wait: sleep
2235
+ }, {
2236
+ delayMs: inputOptions.keyboardTypeDelay
2237
+ });
2238
+ }
2239
+ describe() {
2240
+ const port = this.connectionConfig.port || 3389;
2241
+ const server = formatRdpServerAddress(this.connectionConfig.host, port);
2242
+ const username = this.connectionConfig.username ? ` as ${this.connectionConfig.username}` : '';
2243
+ const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
2244
+ return `RDP Device ${server}${username}${session}`;
2245
+ }
2246
+ async connect() {
2247
+ this.throwIfDestroyed();
2248
+ device_debug('connecting to rdp backend', {
2249
+ host: this.connectionConfig.host,
2250
+ port: this.connectionConfig.port,
2251
+ username: this.connectionConfig.username
2252
+ });
2253
+ this.connectionInfo = await this.backend.connect(this.connectionConfig);
2254
+ this.cursorPosition = [
2255
+ Math.round(this.connectionInfo.size.width / 2),
2256
+ Math.round(this.connectionInfo.size.height / 2)
2257
+ ];
2258
+ }
2259
+ async screenshotBase64() {
2260
+ this.assertConnected();
2261
+ return this.backend.screenshotBase64();
2262
+ }
2263
+ async size() {
2264
+ this.assertConnected();
2265
+ return this.backend.size();
2266
+ }
2267
+ async destroy() {
2268
+ if (this.destroyed) return;
2269
+ this.destroyed = true;
2270
+ this.connectionInfo = void 0;
2271
+ this.cursorPosition = void 0;
2272
+ await this.backend.disconnect();
2273
+ }
2274
+ actionSpace() {
2275
+ const defaultActions = [
2276
+ ...defineActionsFromInputPrimitives(this.inputPrimitives, {
2277
+ size: ()=>this.size()
2278
+ }),
2279
+ defineAction({
2280
+ name: 'ListDisplays',
2281
+ supportedBySystemOne: true,
2282
+ description: 'List all available displays/monitors',
2283
+ call: async ()=>{
2284
+ this.assertConnected();
2285
+ const size = await this.size();
2286
+ const server = this.connectionInfo?.server || formatRdpServerAddress(this.connectionConfig.host, this.connectionConfig.port || 3389);
2287
+ return [
2288
+ {
2289
+ id: this.connectionInfo?.sessionId || this.connectionConfig.host,
2290
+ name: `RDP ${server} (${size.width}x${size.height})`,
2291
+ primary: true
2292
+ }
2293
+ ];
2294
+ }
2295
+ })
2296
+ ];
2297
+ return [
2298
+ ...defaultActions,
2299
+ ...this.customActions
2300
+ ];
2301
+ }
2302
+ assertConnected() {
2303
+ this.throwIfDestroyed();
2304
+ if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
2305
+ }
2306
+ throwIfDestroyed() {
2307
+ if (this.destroyed) throw new Error('RDPDevice has been destroyed');
2308
+ }
2309
+ async moveToElement(element, options) {
2310
+ this.assertConnected();
2311
+ const targetX = Math.round(element.center[0]);
2312
+ const targetY = Math.round(element.center[1]);
2313
+ await this.movePointer(targetX, targetY, options);
2314
+ }
2315
+ async clearInput() {
2316
+ if (this.backend.clearInput) return void await this.backend.clearInput();
2317
+ await this.backend.keyPress('Control+A');
2318
+ await this.backend.keyPress('Backspace');
2319
+ }
2320
+ edgeScrollDirection(scrollType) {
2321
+ switch(scrollType){
2322
+ case 'scrollToTop':
2323
+ return 'up';
2324
+ case 'scrollToBottom':
2325
+ return 'down';
2326
+ case 'scrollToLeft':
2327
+ return 'left';
2328
+ case 'scrollToRight':
2329
+ return 'right';
2330
+ case 'singleAction':
2331
+ return 'down';
2332
+ default:
2333
+ throw new Error(`Unsupported scroll type: ${scrollType}`);
2334
+ }
2335
+ }
2336
+ defaultScrollDistance(direction) {
2337
+ const size = this.connectionInfo?.size;
2338
+ if (!size) return DEFAULT_SCROLL_DISTANCE;
2339
+ const isHorizontal = 'left' === direction || 'right' === direction;
2340
+ const base = isHorizontal ? size.width : size.height;
2341
+ return Math.max(1, Math.round(base * device_DEFAULT_SCROLL_VIEWPORT_RATIO));
2342
+ }
2343
+ async movePointer(targetX, targetY, options) {
2344
+ this.assertConnected();
2345
+ const start = this.cursorPosition || [
2346
+ targetX,
2347
+ targetY
2348
+ ];
2349
+ const steps = Math.max(1, options?.steps || 1);
2350
+ const stepDelayMs = options?.stepDelayMs || 0;
2351
+ for(let step = 1; step <= steps; step++){
2352
+ const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
2353
+ const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
2354
+ await this.backend.mouseMove(x, y);
2355
+ this.cursorPosition = [
2356
+ x,
2357
+ y
2358
+ ];
2359
+ if (stepDelayMs > 0 && step < steps) await sleep(stepDelayMs);
2360
+ }
2361
+ if (options?.settleDelayMs) await sleep(options.settleDelayMs);
2362
+ }
2363
+ async performPointerDrag(from, to, duration) {
2364
+ const screenSize = await this.size();
2365
+ const boundedFrom = clampPointerPointToSize(from, screenSize);
2366
+ const boundedTo = clampPointerPointToSize(to, screenSize);
2367
+ await this.movePointer(Math.round(boundedFrom.x), Math.round(boundedFrom.y), {
2368
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
2369
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
2370
+ });
2371
+ await this.backend.mouseButton('left', 'down');
2372
+ try {
2373
+ await sleep(DRAG_HOLD_DURATION);
2374
+ await this.movePointer(Math.round(boundedTo.x), Math.round(boundedTo.y), {
2375
+ steps: device_SMOOTH_MOVE_STEPS_DRAG,
2376
+ stepDelayMs: void 0 === duration ? device_SMOOTH_MOVE_DELAY_DRAG : Math.max(0, Math.round(duration / device_SMOOTH_MOVE_STEPS_DRAG))
2377
+ });
2378
+ await sleep(DRAG_HOLD_DURATION);
2379
+ } finally{
2380
+ await this.backend.mouseButton('left', 'up');
2381
+ }
2382
+ }
2383
+ async performWheel(direction, amount, x, y) {
2384
+ let remaining = Math.abs(amount);
2385
+ if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
2386
+ while(remaining > 0){
2387
+ const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
2388
+ await this.backend.wheel(direction, chunk, x, y);
2389
+ remaining -= chunk;
2390
+ if (remaining > 0) await sleep(device_SCROLL_STEP_DELAY);
2391
+ }
2392
+ }
2393
+ constructor(options){
2394
+ rdp_device_define_property(this, "interfaceType", 'rdp');
2395
+ rdp_device_define_property(this, "connectionConfig", void 0);
2396
+ rdp_device_define_property(this, "inputOptions", void 0);
2397
+ rdp_device_define_property(this, "customActions", void 0);
2398
+ rdp_device_define_property(this, "backend", void 0);
2399
+ rdp_device_define_property(this, "connectionInfo", void 0);
2400
+ rdp_device_define_property(this, "destroyed", false);
2401
+ rdp_device_define_property(this, "cursorPosition", void 0);
2402
+ rdp_device_define_property(this, "uri", void 0);
2403
+ rdp_device_define_property(this, "inputPrimitives", {
2404
+ pointer: {
2405
+ tap: async ({ x, y })=>{
2406
+ await this.movePointer(Math.round(x), Math.round(y), {
2407
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
2408
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
2409
+ });
2410
+ await this.backend.mouseButton('left', 'down');
2411
+ await sleep(device_CLICK_HOLD_DURATION);
2412
+ await this.backend.mouseButton('left', 'up');
2413
+ },
2414
+ doubleClick: async ({ x, y })=>{
2415
+ await this.movePointer(Math.round(x), Math.round(y), {
2416
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
2417
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
2418
+ });
2419
+ await this.backend.mouseButton('left', 'doubleClick');
2420
+ },
2421
+ rightClick: async ({ x, y })=>{
2422
+ await this.movePointer(Math.round(x), Math.round(y), {
2423
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
2424
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
2425
+ });
2426
+ await this.backend.mouseButton('right', 'click');
2427
+ },
2428
+ middleClick: async ({ x, y })=>{
2429
+ await this.movePointer(Math.round(x), Math.round(y), {
2430
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
2431
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
2432
+ });
2433
+ await this.backend.mouseButton('middle', 'click');
2434
+ },
2435
+ hover: async ({ x, y })=>{
2436
+ await this.movePointer(Math.round(x), Math.round(y), {
2437
+ steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
2438
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
2439
+ settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
2440
+ });
2441
+ },
2442
+ dragAndDrop: async (from, to)=>{
2443
+ await this.performPointerDrag(from, to);
2444
+ },
2445
+ swipe: async (from, to, opts)=>{
2446
+ const repeatCount = opts?.repeat ?? 1;
2447
+ for(let index = 0; index < repeatCount; index++)await this.performPointerDrag(from, to, opts?.duration);
2448
+ }
2449
+ },
2450
+ keyboard: {
2451
+ typeText: async (value, opts)=>{
2452
+ this.assertConnected();
2453
+ const resolvedInputOptions = resolveTextInputOptions(opts, this.inputOptions);
2454
+ const target = opts?.target;
2455
+ if (target) {
2456
+ await this.inputPrimitives.pointer.tap({
2457
+ x: target.center[0],
2458
+ y: target.center[1]
2459
+ });
2460
+ await sleep(device_INPUT_FOCUS_DELAY);
2461
+ }
2462
+ if (opts?.replace !== false) {
2463
+ await this.clearInput();
2464
+ await sleep(device_INPUT_CLEAR_DELAY);
2465
+ }
2466
+ if (opts?.focusOnly || !value) return;
2467
+ await this.typeText(value, resolvedInputOptions);
2468
+ },
2469
+ clearInput: async (target)=>{
2470
+ this.assertConnected();
2471
+ const element = target;
2472
+ if (element) {
2473
+ await this.inputPrimitives.pointer.tap({
2474
+ x: element.center[0],
2475
+ y: element.center[1]
2476
+ });
2477
+ await sleep(device_INPUT_FOCUS_DELAY);
2478
+ }
2479
+ await this.clearInput();
2480
+ await sleep(device_INPUT_CLEAR_DELAY);
2481
+ },
2482
+ keyboardPress: async (keyName, opts)=>{
2483
+ this.assertConnected();
2484
+ const target = opts?.target;
2485
+ if (target) await this.inputPrimitives.pointer.tap({
2486
+ x: target.center[0],
2487
+ y: target.center[1]
2488
+ });
2489
+ await this.backend.keyPress(keyName);
2490
+ }
2491
+ },
2492
+ scroll: {
2493
+ scroll: async (param)=>{
2494
+ this.assertConnected();
2495
+ const target = param.locate;
2496
+ if (target) await this.moveToElement(target, {
2497
+ steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
2498
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
2499
+ });
2500
+ if (param.scrollType && 'singleAction' !== param.scrollType) {
2501
+ const direction = this.edgeScrollDirection(param.scrollType);
2502
+ for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
2503
+ await sleep(device_SCROLL_COMPLETE_DELAY);
2504
+ return;
2505
+ }
2506
+ await this.performWheel(param.direction || 'down', param.distance || this.defaultScrollDistance(param.direction || 'down'), target?.center[0], target?.center[1]);
2507
+ await sleep(device_SCROLL_COMPLETE_DELAY);
2508
+ }
2509
+ }
2510
+ });
2511
+ const { backend, customActions, inputStrategy, keyboardTypeDelay, ...connectionConfig } = options;
2512
+ this.connectionConfig = {
2513
+ port: 3389,
2514
+ securityProtocol: 'auto',
2515
+ ignoreCertificate: false,
2516
+ ...normalizeRdpConnectionConfig(connectionConfig)
2517
+ };
2518
+ this.inputOptions = {
2519
+ inputStrategy,
2520
+ keyboardTypeDelay
2521
+ };
2522
+ this.customActions = customActions ?? [];
2523
+ this.backend = backend || createDefaultRDPBackendClient();
2524
+ }
2525
+ }
2526
+ class ComputerAgent extends Agent {
2527
+ }
2528
+ function createLocalComputerDevice(opts) {
2529
+ return new ComputerDevice({
2530
+ displayId: opts?.displayId,
2531
+ customActions: opts?.customActions,
2532
+ keyboardTypeDelay: opts?.keyboardTypeDelay,
2533
+ inputStrategy: opts?.inputStrategy,
2534
+ keyboardDriver: opts?.keyboardDriver,
2535
+ keyboardModifierDelay: opts?.keyboardModifierDelay,
2536
+ keyboardLayout: opts?.keyboardLayout,
2537
+ headless: opts?.headless,
2538
+ xvfbResolution: opts?.xvfbResolution,
2539
+ keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
2540
+ });
2541
+ }
2542
+ function createRDPComputerDevice(opts) {
2543
+ return new RDPDevice({
2544
+ host: opts.host,
2545
+ port: opts.port,
2546
+ username: opts.username,
2547
+ password: opts.password,
2548
+ domain: opts.domain,
2549
+ localAddress: opts.localAddress,
2550
+ adminSession: opts.adminSession,
2551
+ ignoreCertificate: opts.ignoreCertificate,
2552
+ securityProtocol: opts.securityProtocol,
2553
+ desktopWidth: opts.desktopWidth,
2554
+ desktopHeight: opts.desktopHeight,
2555
+ backend: opts.backend,
2556
+ customActions: opts.customActions,
2557
+ keyboardTypeDelay: opts.keyboardTypeDelay,
2558
+ inputStrategy: opts.inputStrategy
2559
+ });
2560
+ }
2561
+ async function agentForComputer(opts) {
2562
+ const device = createLocalComputerDevice(opts);
2563
+ await device.connect();
2564
+ return new ComputerAgent(device, opts);
2565
+ }
2566
+ const agentFromComputer = agentForComputer;
2567
+ async function agentForRDPComputer(opts) {
2568
+ const device = createRDPComputerDevice(opts);
2569
+ await device.connect();
2570
+ return new ComputerAgent(device, opts);
2571
+ }
2572
+ function agent_tools_define_property(obj, key, value) {
2573
+ if (key in obj) Object.defineProperty(obj, key, {
2574
+ value: value,
2575
+ enumerable: true,
2576
+ configurable: true,
2577
+ writable: true
2578
+ });
2579
+ else obj[key] = value;
2580
+ return obj;
2581
+ }
2582
+ const agent_tools_debug = getDebug('agent-tools:computer');
2583
+ const RDP_SECURITY_PROTOCOLS = [
2584
+ 'auto',
2585
+ 'tls',
2586
+ 'nla',
2587
+ 'rdp'
2588
+ ];
2589
+ const computerInitArgShape = {
2590
+ displayId: z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
2591
+ headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
2592
+ keyboardTypeDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
2593
+ inputStrategy: z["enum"](inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
2594
+ keyboardModifierDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds after each modifier transition and the main key for local libnut keyboard events. Positive values pace shortcuts and the implicit Shift used by uppercase letters during sequential input. Layout-dependent punctuation also requires keyboardLayout. Ignored in RDP mode and by the macOS AppleScript driver.'),
2595
+ keyboardLayout: z["enum"]([
2596
+ 'en-US'
2597
+ ]).optional().describe('Keyboard layout used to resolve layout-dependent shifted punctuation to base keys during sequential local libnut input. Currently only "en-US" is supported. Ignored in RDP mode and by the macOS AppleScript driver.'),
2598
+ host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
2599
+ port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
2600
+ username: z.string().optional().describe('RDP username. Requires host.'),
2601
+ password: z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
2602
+ domain: z.string().optional().describe('RDP domain. Requires host.'),
2603
+ localAddress: z.string().optional().describe('Local source IP address for the RDP TCP connection. Requires host.'),
2604
+ adminSession: z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
2605
+ ignoreCertificate: z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
2606
+ securityProtocol: z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
2607
+ desktopWidth: z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
2608
+ desktopHeight: z.number().optional().describe('Remote desktop height in pixels. Requires host.'),
2609
+ ...agentBehaviorInitArgShape
2610
+ };
2611
+ function adaptComputerInitArgs(extracted) {
2612
+ if (!extracted || 0 === Object.keys(extracted).length) return;
2613
+ if (extracted.host) {
2614
+ const { displayId: _d, headless: _h, keyboardModifierDelay: _s, keyboardLayout: _l, ...rdpFields } = extracted;
2615
+ const host = normalizeRdpHost(extracted.host);
2616
+ return {
2617
+ mode: 'rdp',
2618
+ ...rdpFields,
2619
+ host
2620
+ };
2621
+ }
2622
+ return {
2623
+ mode: 'local',
2624
+ displayId: extracted.displayId,
2625
+ headless: extracted.headless,
2626
+ keyboardTypeDelay: extracted.keyboardTypeDelay,
2627
+ inputStrategy: extracted.inputStrategy,
2628
+ keyboardModifierDelay: extracted.keyboardModifierDelay,
2629
+ keyboardLayout: extracted.keyboardLayout,
2630
+ ...extractAgentBehaviorInitArgs(extracted) ?? {}
2631
+ };
2632
+ }
2633
+ function describeConnectTarget(opts) {
2634
+ if (opts?.mode === 'rdp') {
2635
+ const target = opts.port ? formatRdpServerAddress(opts.host, opts.port) : formatRdpHost(opts.host);
2636
+ const userSuffix = opts.username ? ` as ${opts.username}` : '';
2637
+ return ` via RDP (${target}${userSuffix})`;
2638
+ }
2639
+ if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
2640
+ return ' (Primary display)';
2641
+ }
2642
+ function getCliReportSessionTarget(opts) {
2643
+ if (opts?.mode === 'rdp') return `rdp:${formatRdpHost(opts.host)}`;
2644
+ if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
2645
+ return 'primary';
2646
+ }
2647
+ class ComputerMidsceneTools extends BaseMidsceneTools {
2648
+ getCliReportSessionName() {
2649
+ return 'midscene-computer';
2650
+ }
2651
+ createTemporaryDevice() {
2652
+ return new ComputerDevice({});
2653
+ }
2654
+ async ensureAgent(opts) {
2655
+ const nextSignature = getAgentInitArgsSignature(opts);
2656
+ if (this.agent && shouldRebuildAgentForInitArgs(this.lastInitArgsSignature, nextSignature)) {
2657
+ try {
2658
+ await this.agent.destroy?.();
2659
+ } catch (error) {
2660
+ agent_tools_debug('Failed to destroy agent during cleanup:', error);
2661
+ }
2662
+ this.agent = void 0;
2663
+ }
2664
+ if (this.agent) return this.agent;
2665
+ const reportOptions = this.readCliReportAgentOptions();
2666
+ if (opts?.mode === 'rdp') {
2667
+ agent_tools_debug('Creating RDP Computer agent for host:', opts.host);
2668
+ const { mode: _mode, ...rdpFields } = opts;
2669
+ const agent = await agentForRDPComputer({
2670
+ ...rdpFields,
2671
+ ...reportOptions ?? {}
2672
+ });
2673
+ this.agent = agent;
2674
+ this.lastInitArgsSignature = nextSignature;
2675
+ return agent;
2676
+ }
2677
+ const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
2678
+ const headless = opts?.mode === 'local' ? opts.headless : void 0;
2679
+ const keyboardTypeDelay = opts?.keyboardTypeDelay;
2680
+ const inputStrategy = opts?.inputStrategy;
2681
+ const keyboardModifierDelay = opts?.mode === 'local' ? opts.keyboardModifierDelay : void 0;
2682
+ const keyboardLayout = opts?.mode === 'local' ? opts.keyboardLayout : void 0;
2683
+ agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
2684
+ const agentOpts = {
2685
+ ...displayId ? {
2686
+ displayId
2687
+ } : {},
2688
+ ...void 0 !== headless ? {
2689
+ headless
2690
+ } : {},
2691
+ ...void 0 !== keyboardTypeDelay ? {
2692
+ keyboardTypeDelay
2693
+ } : {},
2694
+ ...void 0 !== inputStrategy ? {
2695
+ inputStrategy
2696
+ } : {},
2697
+ ...void 0 !== keyboardModifierDelay ? {
2698
+ keyboardModifierDelay
2699
+ } : {},
2700
+ ...void 0 !== keyboardLayout ? {
2701
+ keyboardLayout
2702
+ } : {},
2703
+ ...this.options.keepXvfbAliveUntilProcessExit ? {
2704
+ keepXvfbAliveUntilProcessExit: true
2705
+ } : {},
2706
+ ...extractAgentBehaviorInitArgs(opts) ?? {},
2707
+ ...reportOptions ?? {}
2708
+ };
2709
+ const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
2710
+ this.agent = agent;
2711
+ this.lastInitArgsSignature = nextSignature;
2712
+ return agent;
2713
+ }
2714
+ preparePlatformTools() {
2715
+ return [
2716
+ {
2717
+ name: 'computer_connect',
2718
+ description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/localAddress/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
2719
+ schema: this.getAgentInitArgSchema(),
2720
+ cli: this.getAgentInitArgCliMetadata(),
2721
+ handler: async (args)=>{
2722
+ const initArgs = this.extractAgentInitParam(args);
2723
+ const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
2724
+ this.commitCliReportSession(reportSession);
2725
+ if (this.agent) {
2726
+ try {
2727
+ await this.agent.destroy?.();
2728
+ } catch (error) {
2729
+ agent_tools_debug('Failed to destroy agent during connect:', error);
2730
+ }
2731
+ this.agent = void 0;
2732
+ this.lastInitArgsSignature = void 0;
2733
+ }
2734
+ const agent = await this.ensureAgent(initArgs);
2735
+ const screenshot = await agent.interface.screenshotBase64();
2736
+ return {
2737
+ content: [
2738
+ {
2739
+ type: 'text',
2740
+ text: `Connected to computer${describeConnectTarget(initArgs)}`
2741
+ },
2742
+ ...this.buildScreenshotContent(screenshot)
2743
+ ]
2744
+ };
2745
+ }
2746
+ },
2747
+ {
2748
+ name: 'computer_disconnect',
2749
+ description: 'Disconnect from computer and release resources',
2750
+ schema: {},
2751
+ handler: this.createDisconnectHandler('computer')
2752
+ },
2753
+ {
2754
+ name: 'computer_list_displays',
2755
+ description: 'List all available displays/monitors',
2756
+ schema: {},
2757
+ handler: async ()=>{
2758
+ const displays = await ComputerDevice.listDisplays();
2759
+ return {
2760
+ content: [
2761
+ {
2762
+ type: 'text',
2763
+ text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
2764
+ }
2765
+ ]
2766
+ };
2767
+ }
2768
+ }
2769
+ ];
2770
+ }
2771
+ constructor(options = {}){
2772
+ super(), agent_tools_define_property(this, "options", void 0), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", void 0), this.options = options, this.initArgSpec = {
2773
+ namespace: 'computer',
2774
+ shape: computerInitArgShape,
2775
+ cli: {
2776
+ preferBareKeys: true
2777
+ },
2778
+ adapt: (extracted)=>adaptComputerInitArgs(extracted)
2779
+ };
2780
+ }
2781
+ }
2782
+ export { ComputerAgent, ComputerDevice, ComputerMidsceneTools, HelperProcessRDPBackendClient, RDPDevice, UnsupportedRDPBackendClient, agentForComputer, agentForRDPComputer, agentFromComputer, checkXvfbInstalled, createDefaultRDPBackendClient, createReportCliCommands, createRequire, needsXvfb };