@midscene/computer 1.17.8-beta-20261106101231.0 → 1.17.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/430.mjs +2782 -0
- package/dist/es/cli.mjs +2 -2739
- package/dist/es/index.mjs +5 -2784
- package/dist/lib/cli.js +3 -3
- package/dist/lib/index.js +4 -4
- package/package.json +5 -5
package/dist/es/index.mjs
CHANGED
|
@@ -1,2787 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { chmodSync, existsSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
4
|
-
import { createRequire } from "node:module";
|
|
5
|
-
import { tmpdir } from "node:os";
|
|
6
|
-
import { dirname, join, resolve as external_node_path_resolve } from "node:path";
|
|
7
|
-
import { fileURLToPath } from "node:url";
|
|
8
|
-
import { defineAction, defineActionsFromInputPrimitives, inputStrategies, resolveTextInputOptions, sendTextSequentially, shouldInputSequentially } from "@midscene/core/device";
|
|
9
|
-
import { sleep } from "@midscene/core/utils";
|
|
10
|
-
import { createImgBase64ByFormat } from "@midscene/shared/img";
|
|
11
|
-
import { getDebug } from "@midscene/shared/logger";
|
|
12
|
-
import screenshot_desktop from "screenshot-desktop";
|
|
13
|
-
import node_assert from "node:assert";
|
|
14
|
-
import { hasActiveCliInterruptWaiter } from "@midscene/shared/cli/interrupt";
|
|
15
|
-
import { Agent } from "@midscene/core/agent";
|
|
16
|
-
import { once } from "node:events";
|
|
17
|
-
import { createInterface } from "node:readline";
|
|
18
|
-
import { z } from "@midscene/core";
|
|
19
|
-
import { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/agent-tools/agent-behavior-init-args";
|
|
20
|
-
import { BaseMidsceneTools } from "@midscene/shared/agent-tools/base-tools";
|
|
21
|
-
import { overrideAIConfig } from "@midscene/shared/env";
|
|
22
|
-
const debugKeyboard = getDebug('computer:keyboard');
|
|
23
|
-
const APPLE_SCRIPT_KEY_CODES = {
|
|
24
|
-
return: 36,
|
|
25
|
-
enter: 36,
|
|
26
|
-
tab: 48,
|
|
27
|
-
space: 49,
|
|
28
|
-
backspace: 51,
|
|
29
|
-
delete: 51,
|
|
30
|
-
escape: 53,
|
|
31
|
-
forwarddelete: 117,
|
|
32
|
-
left: 123,
|
|
33
|
-
right: 124,
|
|
34
|
-
down: 125,
|
|
35
|
-
up: 126,
|
|
36
|
-
home: 115,
|
|
37
|
-
end: 119,
|
|
38
|
-
pageup: 116,
|
|
39
|
-
pagedown: 121,
|
|
40
|
-
f1: 122,
|
|
41
|
-
f2: 120,
|
|
42
|
-
f3: 99,
|
|
43
|
-
f4: 118,
|
|
44
|
-
f5: 96,
|
|
45
|
-
f6: 97,
|
|
46
|
-
f7: 98,
|
|
47
|
-
f8: 100,
|
|
48
|
-
f9: 101,
|
|
49
|
-
f10: 109,
|
|
50
|
-
f11: 103,
|
|
51
|
-
f12: 111
|
|
52
|
-
};
|
|
53
|
-
const APPLE_SCRIPT_MODIFIER_KEYS = {
|
|
54
|
-
command: 'command',
|
|
55
|
-
cmd: 'command',
|
|
56
|
-
control: 'control',
|
|
57
|
-
ctrl: 'control',
|
|
58
|
-
shift: 'shift',
|
|
59
|
-
alt: 'option',
|
|
60
|
-
option: 'option',
|
|
61
|
-
meta: 'command'
|
|
62
|
-
};
|
|
63
|
-
function buildKeyCommand(key) {
|
|
64
|
-
const keyCode = APPLE_SCRIPT_KEY_CODES[key.toLowerCase()];
|
|
65
|
-
if (void 0 !== keyCode) return `key code ${keyCode}`;
|
|
66
|
-
const escapedKey = key.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
67
|
-
return `keystroke "${escapedKey}"`;
|
|
68
|
-
}
|
|
69
|
-
function resolveModifierKeys(modifiers) {
|
|
70
|
-
return modifiers.map((modifier)=>APPLE_SCRIPT_MODIFIER_KEYS[modifier.toLowerCase()]).filter((modifier)=>void 0 !== modifier);
|
|
71
|
-
}
|
|
72
|
-
function buildAppleScriptKeyPress(key, modifiers = []) {
|
|
73
|
-
const modifierKeys = resolveModifierKeys(modifiers);
|
|
74
|
-
const modifierClause = modifierKeys.length ? ` using {${modifierKeys.map((modifier)=>`${modifier} down`).join(', ')}}` : '';
|
|
75
|
-
return `tell application "System Events" to ${buildKeyCommand(key)}${modifierClause}`;
|
|
76
|
-
}
|
|
77
|
-
function sendKeyViaAppleScript(key, modifiers = []) {
|
|
78
|
-
const script = buildAppleScriptKeyPress(key, modifiers);
|
|
79
|
-
debugKeyboard('sendKeyViaAppleScript', {
|
|
80
|
-
key,
|
|
81
|
-
modifiers,
|
|
82
|
-
script
|
|
83
|
-
});
|
|
84
|
-
execFileSync("osascript", [
|
|
85
|
-
'-e',
|
|
86
|
-
script
|
|
87
|
-
]);
|
|
88
|
-
}
|
|
89
|
-
function _define_property(obj, key, value) {
|
|
90
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
91
|
-
value: value,
|
|
92
|
-
enumerable: true,
|
|
93
|
-
configurable: true,
|
|
94
|
-
writable: true
|
|
95
|
-
});
|
|
96
|
-
else obj[key] = value;
|
|
97
|
-
return obj;
|
|
98
|
-
}
|
|
99
|
-
const MOUSE_COORDINATE_TOLERANCE_PX = 5;
|
|
100
|
-
class ComputerInputDriver {
|
|
101
|
-
destroy() {
|
|
102
|
-
if (this.destroyed) return;
|
|
103
|
-
this.destroyed = true;
|
|
104
|
-
this.rejectPendingInputDelays();
|
|
105
|
-
}
|
|
106
|
-
getScreenSize() {
|
|
107
|
-
return this.getLibnutOrThrow('getScreenSize').getScreenSize();
|
|
108
|
-
}
|
|
109
|
-
getMousePos() {
|
|
110
|
-
return this.getLibnutOrThrow('getMousePos').getMousePos();
|
|
111
|
-
}
|
|
112
|
-
moveMouse(x, y) {
|
|
113
|
-
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
114
|
-
}
|
|
115
|
-
assertMousePosition(targetX, targetY, context) {
|
|
116
|
-
const current = this.getMousePos();
|
|
117
|
-
const drift = {
|
|
118
|
-
x: current.x - targetX,
|
|
119
|
-
y: current.y - targetY
|
|
120
|
-
};
|
|
121
|
-
if (Math.abs(drift.x) > MOUSE_COORDINATE_TOLERANCE_PX || Math.abs(drift.y) > MOUSE_COORDINATE_TOLERANCE_PX) throw new Error(`${context}: expected (${targetX}, ${targetY}), got (${current.x}, ${current.y}), drift=(${drift.x}, ${drift.y})`);
|
|
122
|
-
return drift;
|
|
123
|
-
}
|
|
124
|
-
focusActiveWindow() {
|
|
125
|
-
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
126
|
-
if ('function' != typeof lib.getActiveWindow || 'function' != typeof lib.focusWindow) return false;
|
|
127
|
-
try {
|
|
128
|
-
const handle = lib.getActiveWindow();
|
|
129
|
-
if (!handle) return false;
|
|
130
|
-
lib.focusWindow(handle);
|
|
131
|
-
return true;
|
|
132
|
-
} catch (error) {
|
|
133
|
-
this.options.debug(`focusActiveWindow failed: ${error}`);
|
|
134
|
-
return false;
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
mouseClick(button, double) {
|
|
138
|
-
const lib = this.getLibnutOrThrow('mouseClick');
|
|
139
|
-
if (void 0 !== double) lib.mouseClick(button, double);
|
|
140
|
-
else if (void 0 !== button) lib.mouseClick(button);
|
|
141
|
-
else lib.mouseClick();
|
|
142
|
-
}
|
|
143
|
-
mouseToggle(state, button = 'left') {
|
|
144
|
-
this.getLibnutOrThrow('mouseToggle').mouseToggle(state, button);
|
|
145
|
-
}
|
|
146
|
-
scrollMouse(x, y) {
|
|
147
|
-
this.getLibnutOrThrow('scrollMouse').scrollMouse(x, y);
|
|
148
|
-
}
|
|
149
|
-
async emitScrollDetents(dx, dy, detents, delayMs) {
|
|
150
|
-
this.assertActive('emitScrollDetents');
|
|
151
|
-
for(let i = 0; i < detents; i++){
|
|
152
|
-
this.scrollMouse(dx, dy);
|
|
153
|
-
if (i < detents - 1) await this.delay(delayMs);
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
keyTap(key, modifiers) {
|
|
157
|
-
const lib = this.getLibnutOrThrow('keyTap');
|
|
158
|
-
if (void 0 !== modifiers) lib.keyTap(key, modifiers);
|
|
159
|
-
else lib.keyTap(key);
|
|
160
|
-
}
|
|
161
|
-
keyToggle(key, state, modifiers) {
|
|
162
|
-
const lib = this.getLibnutOrThrow('keyToggle');
|
|
163
|
-
if (void 0 !== modifiers) lib.keyToggle(key, state, modifiers);
|
|
164
|
-
else lib.keyToggle(key, state);
|
|
165
|
-
}
|
|
166
|
-
async keyTapWithModifierDelay(key, modifiers, delayMs) {
|
|
167
|
-
const uniqueModifiers = [
|
|
168
|
-
...new Set(modifiers)
|
|
169
|
-
];
|
|
170
|
-
if (0 === uniqueModifiers.length) return void this.keyTap(key);
|
|
171
|
-
if (0 === delayMs) return void this.keyTap(key, uniqueModifiers);
|
|
172
|
-
const pressedModifiers = [];
|
|
173
|
-
let failure;
|
|
174
|
-
try {
|
|
175
|
-
for (const modifier of uniqueModifiers){
|
|
176
|
-
this.keyToggle(modifier, 'down');
|
|
177
|
-
pressedModifiers.push(modifier);
|
|
178
|
-
await this.delay(delayMs);
|
|
179
|
-
}
|
|
180
|
-
this.keyTap(key);
|
|
181
|
-
await this.delay(delayMs);
|
|
182
|
-
} catch (error) {
|
|
183
|
-
failure = error;
|
|
184
|
-
}
|
|
185
|
-
for (const modifier of pressedModifiers.reverse()){
|
|
186
|
-
const releaseFailure = this.tryReleaseKey(modifier);
|
|
187
|
-
if (void 0 === failure && void 0 !== releaseFailure) failure = releaseFailure;
|
|
188
|
-
if (void 0 === failure) try {
|
|
189
|
-
await this.delay(delayMs);
|
|
190
|
-
} catch (error) {
|
|
191
|
-
failure = error;
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
if (void 0 !== failure) throw failure;
|
|
195
|
-
}
|
|
196
|
-
typeString(text) {
|
|
197
|
-
this.getLibnutOrThrow('typeString').typeString(text);
|
|
198
|
-
}
|
|
199
|
-
sendKeyViaAppleScript(key, modifiers = []) {
|
|
200
|
-
this.assertActive('sendKeyViaAppleScript');
|
|
201
|
-
this.options.sendKeyViaAppleScript(key, modifiers);
|
|
202
|
-
}
|
|
203
|
-
sendKey(key, modifiers = []) {
|
|
204
|
-
if (this.options.useAppleScript()) return void this.sendKeyViaAppleScript(key, modifiers);
|
|
205
|
-
if (modifiers.length > 0) this.keyTap(key, modifiers);
|
|
206
|
-
else this.keyTap(key);
|
|
207
|
-
}
|
|
208
|
-
runPhasedScroll(direction, pixels, steps) {
|
|
209
|
-
this.assertActive('runPhasedScroll');
|
|
210
|
-
return this.options.runPhasedScroll(direction, pixels, steps);
|
|
211
|
-
}
|
|
212
|
-
async delay(ms) {
|
|
213
|
-
this.assertActive('delay');
|
|
214
|
-
return new Promise((resolve, reject)=>{
|
|
215
|
-
const waitRef = {
|
|
216
|
-
timeoutId: setTimeout(()=>{
|
|
217
|
-
this.pendingInputDelayWaits.delete(waitRef);
|
|
218
|
-
try {
|
|
219
|
-
this.assertActive('delay');
|
|
220
|
-
resolve();
|
|
221
|
-
} catch (error) {
|
|
222
|
-
reject(error);
|
|
223
|
-
}
|
|
224
|
-
}, ms),
|
|
225
|
-
reject
|
|
226
|
-
};
|
|
227
|
-
this.pendingInputDelayWaits.add(waitRef);
|
|
228
|
-
});
|
|
229
|
-
}
|
|
230
|
-
async smoothMoveMouse(targetX, targetY, steps, stepDelay) {
|
|
231
|
-
const currentPos = this.getMousePos();
|
|
232
|
-
for(let i = 1; i <= steps; i++){
|
|
233
|
-
const stepX = Math.round(currentPos.x + (targetX - currentPos.x) * i / steps);
|
|
234
|
-
const stepY = Math.round(currentPos.y + (targetY - currentPos.y) * i / steps);
|
|
235
|
-
this.moveMouse(stepX, stepY);
|
|
236
|
-
await this.delay(stepDelay);
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
async withMouseButton(button, run) {
|
|
240
|
-
this.mouseToggle('down', button);
|
|
241
|
-
try {
|
|
242
|
-
return await run();
|
|
243
|
-
} finally{
|
|
244
|
-
this.releaseMouseButton(button);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
getLibnutOrThrow(methodName) {
|
|
248
|
-
this.assertActive(methodName);
|
|
249
|
-
const libnut = this.options.getLibnut();
|
|
250
|
-
node_assert(libnut, 'libnut not initialized');
|
|
251
|
-
return libnut;
|
|
252
|
-
}
|
|
253
|
-
assertActive(methodName) {
|
|
254
|
-
if (this.destroyed) throw this.createDestroyedError(methodName);
|
|
255
|
-
}
|
|
256
|
-
createDestroyedError(methodName) {
|
|
257
|
-
return new Error(`ComputerDevice has been destroyed (cannot run ${methodName})`);
|
|
258
|
-
}
|
|
259
|
-
releaseMouseButton(button) {
|
|
260
|
-
try {
|
|
261
|
-
const libnut = this.options.getLibnut();
|
|
262
|
-
node_assert(libnut, 'libnut not initialized');
|
|
263
|
-
libnut.mouseToggle('up', button);
|
|
264
|
-
} catch (error) {
|
|
265
|
-
this.options.debug(`Failed to release mouse button ${button}: ${error}`);
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
tryReleaseKey(key) {
|
|
269
|
-
try {
|
|
270
|
-
const libnut = this.options.getLibnut();
|
|
271
|
-
node_assert(libnut, 'libnut not initialized');
|
|
272
|
-
libnut.keyToggle(key, 'up');
|
|
273
|
-
return;
|
|
274
|
-
} catch (error) {
|
|
275
|
-
const releaseError = new Error(`Failed to release modifier key "${key}"`, {
|
|
276
|
-
cause: error
|
|
277
|
-
});
|
|
278
|
-
this.options.debug(`${releaseError.message}: ${error}`);
|
|
279
|
-
return releaseError;
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
rejectPendingInputDelays() {
|
|
283
|
-
const error = this.createDestroyedError('in-flight input');
|
|
284
|
-
for (const waitRef of this.pendingInputDelayWaits){
|
|
285
|
-
clearTimeout(waitRef.timeoutId);
|
|
286
|
-
waitRef.reject(error);
|
|
287
|
-
}
|
|
288
|
-
this.pendingInputDelayWaits.clear();
|
|
289
|
-
}
|
|
290
|
-
constructor(options){
|
|
291
|
-
_define_property(this, "options", void 0);
|
|
292
|
-
_define_property(this, "destroyed", void 0);
|
|
293
|
-
_define_property(this, "pendingInputDelayWaits", void 0);
|
|
294
|
-
this.options = options;
|
|
295
|
-
this.destroyed = false;
|
|
296
|
-
this.pendingInputDelayWaits = new Set();
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
const US_SHIFTED_CHARACTER_KEYS = new Map([
|
|
300
|
-
[
|
|
301
|
-
'~',
|
|
302
|
-
'`'
|
|
303
|
-
],
|
|
304
|
-
[
|
|
305
|
-
'!',
|
|
306
|
-
'1'
|
|
307
|
-
],
|
|
308
|
-
[
|
|
309
|
-
'@',
|
|
310
|
-
'2'
|
|
311
|
-
],
|
|
312
|
-
[
|
|
313
|
-
'#',
|
|
314
|
-
'3'
|
|
315
|
-
],
|
|
316
|
-
[
|
|
317
|
-
'$',
|
|
318
|
-
'4'
|
|
319
|
-
],
|
|
320
|
-
[
|
|
321
|
-
'%',
|
|
322
|
-
'5'
|
|
323
|
-
],
|
|
324
|
-
[
|
|
325
|
-
'^',
|
|
326
|
-
'6'
|
|
327
|
-
],
|
|
328
|
-
[
|
|
329
|
-
'&',
|
|
330
|
-
'7'
|
|
331
|
-
],
|
|
332
|
-
[
|
|
333
|
-
'*',
|
|
334
|
-
'8'
|
|
335
|
-
],
|
|
336
|
-
[
|
|
337
|
-
'(',
|
|
338
|
-
'9'
|
|
339
|
-
],
|
|
340
|
-
[
|
|
341
|
-
')',
|
|
342
|
-
'0'
|
|
343
|
-
],
|
|
344
|
-
[
|
|
345
|
-
'_',
|
|
346
|
-
'-'
|
|
347
|
-
],
|
|
348
|
-
[
|
|
349
|
-
'+',
|
|
350
|
-
'='
|
|
351
|
-
],
|
|
352
|
-
[
|
|
353
|
-
'{',
|
|
354
|
-
'['
|
|
355
|
-
],
|
|
356
|
-
[
|
|
357
|
-
'}',
|
|
358
|
-
']'
|
|
359
|
-
],
|
|
360
|
-
[
|
|
361
|
-
'|',
|
|
362
|
-
'\\'
|
|
363
|
-
],
|
|
364
|
-
[
|
|
365
|
-
':',
|
|
366
|
-
';'
|
|
367
|
-
],
|
|
368
|
-
[
|
|
369
|
-
'"',
|
|
370
|
-
"'"
|
|
371
|
-
],
|
|
372
|
-
[
|
|
373
|
-
'<',
|
|
374
|
-
','
|
|
375
|
-
],
|
|
376
|
-
[
|
|
377
|
-
'>',
|
|
378
|
-
'.'
|
|
379
|
-
],
|
|
380
|
-
[
|
|
381
|
-
'?',
|
|
382
|
-
'/'
|
|
383
|
-
]
|
|
384
|
-
]);
|
|
385
|
-
function resolveShiftedKey(character, keyboardLayout) {
|
|
386
|
-
if (/^[A-Z]$/.test(character)) return character.toLowerCase();
|
|
387
|
-
return 'en-US' === keyboardLayout ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
|
|
388
|
-
}
|
|
389
|
-
function clampPointerPointToSize(point, size) {
|
|
390
|
-
const maximumX = Math.max(0, Math.floor(size.width) - 1);
|
|
391
|
-
const maximumY = Math.max(0, Math.floor(size.height) - 1);
|
|
392
|
-
return {
|
|
393
|
-
x: Math.max(0, Math.min(point.x, maximumX)),
|
|
394
|
-
y: Math.max(0, Math.min(point.y, maximumY))
|
|
395
|
-
};
|
|
396
|
-
}
|
|
397
|
-
const POWERSHELL_TIMEOUT_MS = 15000;
|
|
398
|
-
const POWERSHELL_MAX_BUFFER = 67108864;
|
|
399
|
-
const WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE = `
|
|
400
|
-
$midsceneAssemblyName = New-Object System.Reflection.AssemblyName('MidsceneDpiNative')
|
|
401
|
-
$midsceneAssembly = [System.AppDomain]::CurrentDomain.DefineDynamicAssembly(
|
|
402
|
-
$midsceneAssemblyName,
|
|
403
|
-
[System.Reflection.Emit.AssemblyBuilderAccess]::Run
|
|
404
|
-
)
|
|
405
|
-
$midsceneModule = $midsceneAssembly.DefineDynamicModule('MidsceneDpiNativeModule')
|
|
406
|
-
$midsceneType = $midsceneModule.DefineType(
|
|
407
|
-
'MidsceneDpiNative.User32',
|
|
408
|
-
[System.Reflection.TypeAttributes]'Public, Class, Sealed, Abstract'
|
|
409
|
-
)
|
|
410
|
-
$midsceneMethodAttributes =
|
|
411
|
-
[System.Reflection.MethodAttributes]::Public -bor
|
|
412
|
-
[System.Reflection.MethodAttributes]::Static -bor
|
|
413
|
-
[System.Reflection.MethodAttributes]::PinvokeImpl
|
|
414
|
-
$midsceneSetDpiMethod = $midsceneType.DefinePInvokeMethod(
|
|
415
|
-
'SetThreadDpiAwarenessContext',
|
|
416
|
-
'user32.dll',
|
|
417
|
-
$midsceneMethodAttributes,
|
|
418
|
-
[System.Reflection.CallingConventions]::Standard,
|
|
419
|
-
[System.IntPtr],
|
|
420
|
-
[System.Type[]]@([System.IntPtr]),
|
|
421
|
-
[System.Runtime.InteropServices.CallingConvention]::Winapi,
|
|
422
|
-
[System.Runtime.InteropServices.CharSet]::None
|
|
423
|
-
)
|
|
424
|
-
$midsceneSetDpiMethod.SetImplementationFlags(
|
|
425
|
-
$midsceneSetDpiMethod.GetMethodImplementationFlags() -bor
|
|
426
|
-
[System.Reflection.MethodImplAttributes]::PreserveSig
|
|
427
|
-
)
|
|
428
|
-
$midsceneGetForegroundWindowMethod = $midsceneType.DefinePInvokeMethod(
|
|
429
|
-
'GetForegroundWindow',
|
|
430
|
-
'user32.dll',
|
|
431
|
-
$midsceneMethodAttributes,
|
|
432
|
-
[System.Reflection.CallingConventions]::Standard,
|
|
433
|
-
[System.IntPtr],
|
|
434
|
-
[System.Type[]]@(),
|
|
435
|
-
[System.Runtime.InteropServices.CallingConvention]::Winapi,
|
|
436
|
-
[System.Runtime.InteropServices.CharSet]::None
|
|
437
|
-
)
|
|
438
|
-
$midsceneGetForegroundWindowMethod.SetImplementationFlags(
|
|
439
|
-
$midsceneGetForegroundWindowMethod.GetMethodImplementationFlags() -bor
|
|
440
|
-
[System.Reflection.MethodImplAttributes]::PreserveSig
|
|
441
|
-
)
|
|
442
|
-
$midsceneGetWindowRectMethod = $midsceneType.DefinePInvokeMethod(
|
|
443
|
-
'GetWindowRect',
|
|
444
|
-
'user32.dll',
|
|
445
|
-
$midsceneMethodAttributes,
|
|
446
|
-
[System.Reflection.CallingConventions]::Standard,
|
|
447
|
-
[bool],
|
|
448
|
-
[System.Type[]]@([System.IntPtr], [System.IntPtr]),
|
|
449
|
-
[System.Runtime.InteropServices.CallingConvention]::Winapi,
|
|
450
|
-
[System.Runtime.InteropServices.CharSet]::None
|
|
451
|
-
)
|
|
452
|
-
$midsceneGetWindowRectMethod.SetImplementationFlags(
|
|
453
|
-
$midsceneGetWindowRectMethod.GetMethodImplementationFlags() -bor
|
|
454
|
-
[System.Reflection.MethodImplAttributes]::PreserveSig
|
|
455
|
-
)
|
|
456
|
-
$midsceneNativeMethods = $midsceneType.CreateType()
|
|
457
|
-
$midscenePerMonitorV2 = [System.IntPtr](-4)
|
|
458
|
-
$midscenePreviousDpiContext =
|
|
459
|
-
$midsceneNativeMethods::SetThreadDpiAwarenessContext($midscenePerMonitorV2)
|
|
460
|
-
if ($midscenePreviousDpiContext -eq [System.IntPtr]::Zero) {
|
|
461
|
-
throw 'Unable to enter the Per-Monitor V2 DPI awareness context.'
|
|
462
|
-
}
|
|
463
|
-
`.trim();
|
|
464
|
-
function runWindowsPhysicalPixelPowershell(script) {
|
|
465
|
-
const physicalPixelScript = `$ProgressPreference = 'SilentlyContinue'
|
|
466
|
-
$ErrorActionPreference = 'Stop'
|
|
467
|
-
${WINDOWS_PHYSICAL_PIXEL_POWERSHELL_PREAMBLE}
|
|
468
|
-
${script}`;
|
|
469
|
-
return execFileSync('powershell.exe', [
|
|
470
|
-
'-NoProfile',
|
|
471
|
-
'-Command',
|
|
472
|
-
physicalPixelScript
|
|
473
|
-
], {
|
|
474
|
-
encoding: 'utf8',
|
|
475
|
-
timeout: POWERSHELL_TIMEOUT_MS,
|
|
476
|
-
maxBuffer: POWERSHELL_MAX_BUFFER,
|
|
477
|
-
windowsHide: true
|
|
478
|
-
});
|
|
479
|
-
}
|
|
480
|
-
function isFiniteNumber(value) {
|
|
481
|
-
return 'number' == typeof value && Number.isFinite(value);
|
|
482
|
-
}
|
|
483
|
-
function isWindowsDisplayGeometry(value) {
|
|
484
|
-
if (!value || 'object' != typeof value) return false;
|
|
485
|
-
const candidate = value;
|
|
486
|
-
const bounds = candidate.bounds;
|
|
487
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && !!bounds && isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
488
|
-
}
|
|
489
|
-
function readWindowsDisplayGeometries() {
|
|
490
|
-
const script = `
|
|
491
|
-
Add-Type -AssemblyName System.Windows.Forms
|
|
492
|
-
$screens = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
493
|
-
$bounds = $_.Bounds
|
|
494
|
-
[PSCustomObject]@{
|
|
495
|
-
id = $_.DeviceName
|
|
496
|
-
name = $_.DeviceName
|
|
497
|
-
primary = $_.Primary
|
|
498
|
-
bounds = [PSCustomObject]@{
|
|
499
|
-
x = $bounds.X
|
|
500
|
-
y = $bounds.Y
|
|
501
|
-
width = $bounds.Width
|
|
502
|
-
height = $bounds.Height
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
})
|
|
506
|
-
[Console]::Out.Write((ConvertTo-Json -InputObject $screens -Compress))
|
|
507
|
-
`.trim();
|
|
508
|
-
const output = runWindowsPhysicalPixelPowershell(script).trim();
|
|
509
|
-
if (!output) throw new Error('Windows display enumeration returned no data');
|
|
510
|
-
const parsed = JSON.parse(output);
|
|
511
|
-
if (!Array.isArray(parsed)) throw new Error('Windows display enumeration returned invalid data');
|
|
512
|
-
const displays = parsed.filter(isWindowsDisplayGeometry);
|
|
513
|
-
if (displays.length !== parsed.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
514
|
-
if (0 === displays.length) throw new Error('Windows display enumeration returned no displays');
|
|
515
|
-
return displays;
|
|
516
|
-
}
|
|
517
|
-
function resolveWindowsDisplayGeometryFromList(displayId, displays) {
|
|
518
|
-
if (!displays.length) return;
|
|
519
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays[0];
|
|
520
|
-
return displays.find((display)=>display.id === displayId);
|
|
521
|
-
}
|
|
522
|
-
function windows_pointer_define_property(obj, key, value) {
|
|
523
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
524
|
-
value: value,
|
|
525
|
-
enumerable: true,
|
|
526
|
-
configurable: true,
|
|
527
|
-
writable: true
|
|
528
|
-
});
|
|
529
|
-
else obj[key] = value;
|
|
530
|
-
return obj;
|
|
531
|
-
}
|
|
532
|
-
const WINDOWS_POINTER_TOLERANCE_PX = 5;
|
|
533
|
-
function assertFinitePoint(point, context) {
|
|
534
|
-
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) throw new Error(`${context} must contain finite coordinates, got (${point.x}, ${point.y})`);
|
|
535
|
-
}
|
|
536
|
-
function parseWindowsPointerPosition(output, context) {
|
|
537
|
-
const match = output.trim().match(/^(-?\d+),(-?\d+)$/);
|
|
538
|
-
if (!match) throw new Error(`${context} returned an invalid cursor position: ${JSON.stringify(output.trim())}`);
|
|
539
|
-
return {
|
|
540
|
-
x: Number(match[1]),
|
|
541
|
-
y: Number(match[2])
|
|
542
|
-
};
|
|
543
|
-
}
|
|
544
|
-
function windowsPointerPositionScript() {
|
|
545
|
-
return `
|
|
546
|
-
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
547
|
-
$position = [System.Windows.Forms.Cursor]::Position
|
|
548
|
-
[Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
|
|
549
|
-
`.trim();
|
|
550
|
-
}
|
|
551
|
-
function windowsPointerMoveScript(point, options) {
|
|
552
|
-
assertFinitePoint(point, 'Windows pointer target');
|
|
553
|
-
if (options?.smoothSteps !== void 0 && !Number.isFinite(options.smoothSteps) || options?.smoothDelayMs !== void 0 && !Number.isFinite(options.smoothDelayMs)) throw new Error('Windows pointer smoothing options must be finite numbers');
|
|
554
|
-
const targetX = Math.round(point.x);
|
|
555
|
-
const targetY = Math.round(point.y);
|
|
556
|
-
const smoothSteps = Math.max(1, Math.round(options?.smoothSteps ?? 1));
|
|
557
|
-
const smoothDelayMs = Math.max(0, Math.round(options?.smoothDelayMs ?? 0));
|
|
558
|
-
return `
|
|
559
|
-
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
560
|
-
$targetX = ${targetX}
|
|
561
|
-
$targetY = ${targetY}
|
|
562
|
-
$smoothSteps = ${smoothSteps}
|
|
563
|
-
$smoothDelayMs = ${smoothDelayMs}
|
|
564
|
-
$start = [System.Windows.Forms.Cursor]::Position
|
|
565
|
-
for ($step = 1; $step -le $smoothSteps; $step += 1) {
|
|
566
|
-
$x = [int][Math]::Round($start.X + (($targetX - $start.X) * $step / [double]$smoothSteps))
|
|
567
|
-
$y = [int][Math]::Round($start.Y + (($targetY - $start.Y) * $step / [double]$smoothSteps))
|
|
568
|
-
[System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($x, $y)
|
|
569
|
-
if ($smoothDelayMs -gt 0 -and $step -lt $smoothSteps) {
|
|
570
|
-
Start-Sleep -Milliseconds $smoothDelayMs
|
|
571
|
-
}
|
|
572
|
-
}
|
|
573
|
-
[System.Windows.Forms.Cursor]::Position = [System.Drawing.Point]::new($targetX, $targetY)
|
|
574
|
-
$position = [System.Windows.Forms.Cursor]::Position
|
|
575
|
-
[Console]::Out.Write(('{0},{1}' -f $position.X, $position.Y))
|
|
576
|
-
`.trim();
|
|
577
|
-
}
|
|
578
|
-
function parseWindowsWindowRect(output) {
|
|
579
|
-
const value = output.trim();
|
|
580
|
-
if (!value) return null;
|
|
581
|
-
const match = value.match(/^(-?\d+),(-?\d+),(\d+),(\d+)$/);
|
|
582
|
-
if (!match) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
|
|
583
|
-
const rect = {
|
|
584
|
-
x: Number(match[1]),
|
|
585
|
-
y: Number(match[2]),
|
|
586
|
-
width: Number(match[3]),
|
|
587
|
-
height: Number(match[4])
|
|
588
|
-
};
|
|
589
|
-
if (rect.width <= 0 || rect.height <= 0) throw new Error(`Windows active window query returned an invalid rectangle: ${JSON.stringify(value)}`);
|
|
590
|
-
return rect;
|
|
591
|
-
}
|
|
592
|
-
function windowsActiveWindowRectScript() {
|
|
593
|
-
return `
|
|
594
|
-
$midsceneWindowHandle = $midsceneNativeMethods::GetForegroundWindow()
|
|
595
|
-
if ($midsceneWindowHandle -eq [System.IntPtr]::Zero) {
|
|
596
|
-
return
|
|
597
|
-
}
|
|
598
|
-
$midsceneWindowRectBuffer = [System.Runtime.InteropServices.Marshal]::AllocHGlobal(16)
|
|
599
|
-
try {
|
|
600
|
-
if (-not $midsceneNativeMethods::GetWindowRect($midsceneWindowHandle, $midsceneWindowRectBuffer)) {
|
|
601
|
-
throw 'GetWindowRect failed for the foreground window.'
|
|
602
|
-
}
|
|
603
|
-
$left = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 0)
|
|
604
|
-
$top = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 4)
|
|
605
|
-
$right = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 8)
|
|
606
|
-
$bottom = [System.Runtime.InteropServices.Marshal]::ReadInt32($midsceneWindowRectBuffer, 12)
|
|
607
|
-
[Console]::Out.Write(('{0},{1},{2},{3}' -f $left, $top, ($right - $left), ($bottom - $top)))
|
|
608
|
-
}
|
|
609
|
-
finally {
|
|
610
|
-
[System.Runtime.InteropServices.Marshal]::FreeHGlobal($midsceneWindowRectBuffer)
|
|
611
|
-
}
|
|
612
|
-
`.trim();
|
|
613
|
-
}
|
|
614
|
-
class WindowsPointerDriver {
|
|
615
|
-
getPosition() {
|
|
616
|
-
return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerPositionScript()), 'Windows pointer query');
|
|
617
|
-
}
|
|
618
|
-
getActiveWindowRect() {
|
|
619
|
-
return parseWindowsWindowRect(this.options.runPhysicalPixelPowershell(windowsActiveWindowRectScript()));
|
|
620
|
-
}
|
|
621
|
-
moveTo(point, options) {
|
|
622
|
-
return parseWindowsPointerPosition(this.options.runPhysicalPixelPowershell(windowsPointerMoveScript(point, options)), 'Windows pointer move');
|
|
623
|
-
}
|
|
624
|
-
constructor(options){
|
|
625
|
-
windows_pointer_define_property(this, "options", void 0);
|
|
626
|
-
this.options = options;
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
|
-
function windowsPointerDrift(expected, actual) {
|
|
630
|
-
return {
|
|
631
|
-
x: actual.x - expected.x,
|
|
632
|
-
y: actual.y - expected.y
|
|
633
|
-
};
|
|
634
|
-
}
|
|
635
|
-
function windowsPointerIsWithinTolerance(drift) {
|
|
636
|
-
return Math.abs(drift.x) <= WINDOWS_POINTER_TOLERANCE_PX && Math.abs(drift.y) <= WINDOWS_POINTER_TOLERANCE_PX;
|
|
637
|
-
}
|
|
638
|
-
const debugXvfb = getDebug('computer:xvfb');
|
|
639
|
-
const xvfbCleanupMonitorScript = String.raw`
|
|
640
|
-
const parentPid = Number(process.argv[1]);
|
|
641
|
-
const xvfbPid = Number(process.argv[2]);
|
|
642
|
-
const timer = setInterval(() => {
|
|
643
|
-
try {
|
|
644
|
-
process.kill(xvfbPid, 0);
|
|
645
|
-
} catch {
|
|
646
|
-
clearInterval(timer);
|
|
647
|
-
process.exit(0);
|
|
648
|
-
}
|
|
649
|
-
try {
|
|
650
|
-
process.kill(parentPid, 0);
|
|
651
|
-
return;
|
|
652
|
-
} catch {
|
|
653
|
-
// The owner is gone, so its X11 clients can no longer receive XIO errors.
|
|
654
|
-
}
|
|
655
|
-
try {
|
|
656
|
-
process.kill(xvfbPid, 'SIGTERM');
|
|
657
|
-
} catch {
|
|
658
|
-
// Xvfb may have already exited.
|
|
659
|
-
}
|
|
660
|
-
clearInterval(timer);
|
|
661
|
-
}, 100);
|
|
662
|
-
`;
|
|
663
|
-
function scheduleXvfbStopAfterProcessExit(instance, parentPid = process.pid) {
|
|
664
|
-
const xvfbPid = instance.process.pid;
|
|
665
|
-
if (!xvfbPid) throw new Error('Cannot schedule Xvfb cleanup before its process starts');
|
|
666
|
-
const monitor = spawn(process.execPath, [
|
|
667
|
-
'-e',
|
|
668
|
-
xvfbCleanupMonitorScript,
|
|
669
|
-
String(parentPid),
|
|
670
|
-
String(xvfbPid)
|
|
671
|
-
], {
|
|
672
|
-
detached: true,
|
|
673
|
-
stdio: 'ignore'
|
|
674
|
-
});
|
|
675
|
-
monitor.on('error', (error)=>{
|
|
676
|
-
debugXvfb(`Xvfb cleanup monitor failed: ${error.message}`);
|
|
677
|
-
});
|
|
678
|
-
instance.process.unref();
|
|
679
|
-
monitor.unref();
|
|
680
|
-
return monitor;
|
|
681
|
-
}
|
|
682
|
-
function createXvfbSignalCleanup(cleanup, source = process) {
|
|
683
|
-
return ()=>{
|
|
684
|
-
if (!hasActiveCliInterruptWaiter(source)) cleanup();
|
|
685
|
-
};
|
|
686
|
-
}
|
|
687
|
-
function checkXvfbInstalled() {
|
|
688
|
-
try {
|
|
689
|
-
execSync('which Xvfb', {
|
|
690
|
-
stdio: 'ignore'
|
|
691
|
-
});
|
|
692
|
-
return true;
|
|
693
|
-
} catch {
|
|
694
|
-
return false;
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
function findAvailableDisplay(startFrom = 99) {
|
|
698
|
-
for(let n = startFrom; n < startFrom + 100; n++)if (!existsSync(`/tmp/.X${n}-lock`)) return n;
|
|
699
|
-
throw new Error(`No available display number found (checked ${startFrom} to ${startFrom + 99})`);
|
|
700
|
-
}
|
|
701
|
-
function startXvfb(options) {
|
|
702
|
-
const resolution = options?.resolution || '1920x1080x24';
|
|
703
|
-
const displayNum = options?.displayNumber ?? findAvailableDisplay();
|
|
704
|
-
const display = `:${displayNum}`;
|
|
705
|
-
return new Promise((resolve, reject)=>{
|
|
706
|
-
debugXvfb(`Starting Xvfb on display ${display} with resolution ${resolution}`);
|
|
707
|
-
const xvfbProcess = spawn('Xvfb', [
|
|
708
|
-
display,
|
|
709
|
-
'-screen',
|
|
710
|
-
'0',
|
|
711
|
-
resolution,
|
|
712
|
-
'-ac',
|
|
713
|
-
'-nolisten',
|
|
714
|
-
'tcp'
|
|
715
|
-
], {
|
|
716
|
-
stdio: 'ignore'
|
|
717
|
-
});
|
|
718
|
-
let settled = false;
|
|
719
|
-
xvfbProcess.on('error', (err)=>{
|
|
720
|
-
if (!settled) {
|
|
721
|
-
settled = true;
|
|
722
|
-
reject(new Error(`Failed to start Xvfb: ${err.message}`));
|
|
723
|
-
}
|
|
724
|
-
});
|
|
725
|
-
xvfbProcess.on('exit', (code)=>{
|
|
726
|
-
if (!settled) {
|
|
727
|
-
settled = true;
|
|
728
|
-
reject(new Error(`Xvfb exited unexpectedly with code ${code}`));
|
|
729
|
-
}
|
|
730
|
-
});
|
|
731
|
-
const instance = {
|
|
732
|
-
process: xvfbProcess,
|
|
733
|
-
display,
|
|
734
|
-
stop () {
|
|
735
|
-
try {
|
|
736
|
-
xvfbProcess.kill('SIGTERM');
|
|
737
|
-
} catch {}
|
|
738
|
-
}
|
|
739
|
-
};
|
|
740
|
-
setTimeout(()=>{
|
|
741
|
-
if (!settled) {
|
|
742
|
-
settled = true;
|
|
743
|
-
debugXvfb(`Xvfb started on display ${display}`);
|
|
744
|
-
resolve(instance);
|
|
745
|
-
}
|
|
746
|
-
}, 500);
|
|
747
|
-
});
|
|
748
|
-
}
|
|
749
|
-
function needsXvfb(explicitOpt) {
|
|
750
|
-
if ('linux' !== process.platform) return false;
|
|
751
|
-
return true === explicitOpt;
|
|
752
|
-
}
|
|
753
|
-
function device_define_property(obj, key, value) {
|
|
754
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
755
|
-
value: value,
|
|
756
|
-
enumerable: true,
|
|
757
|
-
configurable: true,
|
|
758
|
-
writable: true
|
|
759
|
-
});
|
|
760
|
-
else obj[key] = value;
|
|
761
|
-
return obj;
|
|
762
|
-
}
|
|
763
|
-
async function captureScreenshot(options) {
|
|
764
|
-
if ('linux' !== process.platform) return screenshot_desktop(options);
|
|
765
|
-
const filename = join(tmpdir(), `midscene-screenshot-${randomUUID()}.png`);
|
|
766
|
-
try {
|
|
767
|
-
await screenshot_desktop({
|
|
768
|
-
...options,
|
|
769
|
-
filename
|
|
770
|
-
});
|
|
771
|
-
return readFileSync(filename);
|
|
772
|
-
} finally{
|
|
773
|
-
rmSync(filename, {
|
|
774
|
-
force: true
|
|
775
|
-
});
|
|
776
|
-
}
|
|
777
|
-
}
|
|
778
|
-
const SMOOTH_MOVE_STEPS_TAP = 8;
|
|
779
|
-
const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
780
|
-
const SMOOTH_MOVE_STEPS_DRAG = 20;
|
|
781
|
-
const SMOOTH_MOVE_DELAY_TAP = 8;
|
|
782
|
-
const SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
783
|
-
const SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
784
|
-
const MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
785
|
-
const CLICK_SETTLE_DELAY = 50;
|
|
786
|
-
const CLICK_HOLD_DURATION = 100;
|
|
787
|
-
const CLICK_FOCUS_SETTLE_DELAY = 120;
|
|
788
|
-
const INPUT_FOCUS_DELAY = 300;
|
|
789
|
-
const INPUT_CLEAR_DELAY = 150;
|
|
790
|
-
const SCROLL_STEP_DELAY = 100;
|
|
791
|
-
const SCROLL_COMPLETE_DELAY = 500;
|
|
792
|
-
const EDGE_SCROLL_TOTAL_PX = 50000;
|
|
793
|
-
const EDGE_SCROLL_STEPS = 400;
|
|
794
|
-
const PHASED_PIXELS_PER_STEP = 30;
|
|
795
|
-
const PHASED_MIN_STEPS = 10;
|
|
796
|
-
const LIBNUT_FALLBACK_PIXELS_PER_DETENT = 100;
|
|
797
|
-
const LIBNUT_FALLBACK_TICK_DELAY_MS = 30;
|
|
798
|
-
const LIBNUT_FALLBACK_MAX_DETENTS = 200;
|
|
799
|
-
const LIBNUT_FALLBACK_DETENT_AMOUNT = 'win32' === process.platform ? 120 : 1;
|
|
800
|
-
const LIBNUT_FALLBACK_EDGE_DETENTS = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(EDGE_SCROLL_TOTAL_PX / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
801
|
-
const DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
802
|
-
const EDGE_SCROLL_SPEC = {
|
|
803
|
-
scrollToTop: {
|
|
804
|
-
direction: 'up',
|
|
805
|
-
key: 'home',
|
|
806
|
-
libnut: [
|
|
807
|
-
0,
|
|
808
|
-
1
|
|
809
|
-
]
|
|
810
|
-
},
|
|
811
|
-
scrollToBottom: {
|
|
812
|
-
direction: 'down',
|
|
813
|
-
key: 'end',
|
|
814
|
-
libnut: [
|
|
815
|
-
0,
|
|
816
|
-
-1
|
|
817
|
-
]
|
|
818
|
-
},
|
|
819
|
-
scrollToLeft: {
|
|
820
|
-
direction: 'left',
|
|
821
|
-
key: 'home',
|
|
822
|
-
libnut: [
|
|
823
|
-
-1,
|
|
824
|
-
0
|
|
825
|
-
]
|
|
826
|
-
},
|
|
827
|
-
scrollToRight: {
|
|
828
|
-
direction: 'right',
|
|
829
|
-
key: 'end',
|
|
830
|
-
libnut: [
|
|
831
|
-
1,
|
|
832
|
-
0
|
|
833
|
-
]
|
|
834
|
-
}
|
|
835
|
-
};
|
|
836
|
-
function escapePowershellSingleQuoted(value) {
|
|
837
|
-
return value.replace(/'/g, "''");
|
|
838
|
-
}
|
|
839
|
-
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
840
|
-
return geometries.map((display)=>({
|
|
841
|
-
id: display.id,
|
|
842
|
-
name: display.name,
|
|
843
|
-
primary: display.primary
|
|
844
|
-
}));
|
|
845
|
-
}
|
|
846
|
-
let device_libnut = null;
|
|
847
|
-
let libnutLoadError = null;
|
|
848
|
-
async function getLibnut() {
|
|
849
|
-
if (device_libnut) return device_libnut;
|
|
850
|
-
if (libnutLoadError) throw libnutLoadError;
|
|
851
|
-
try {
|
|
852
|
-
const require = createRequire(import.meta.url);
|
|
853
|
-
const libnutModule = require('@computer-use/libnut/dist/import_libnut');
|
|
854
|
-
device_libnut = libnutModule.libnut;
|
|
855
|
-
if (!device_libnut) throw new Error('libnut module loaded but libnut object is undefined');
|
|
856
|
-
return device_libnut;
|
|
857
|
-
} catch (error) {
|
|
858
|
-
libnutLoadError = error;
|
|
859
|
-
throw new Error(`Failed to load @computer-use/libnut. Make sure it is properly installed and compiled for your platform. Error: ${error}`);
|
|
860
|
-
}
|
|
861
|
-
}
|
|
862
|
-
const debugDevice = getDebug('computer:device');
|
|
863
|
-
const warnDevice = getDebug('computer:device', {
|
|
864
|
-
console: true
|
|
865
|
-
});
|
|
866
|
-
const debugComputerInput = getDebug('computer:input', {
|
|
867
|
-
console: true
|
|
868
|
-
});
|
|
869
|
-
const WINDOWS_UIPI_DOC_URL = 'https://midscenejs.com/computer-getting-started#windows-clicks-have-no-effect-on-some-apps';
|
|
870
|
-
function resolvePackageRoot(helperName) {
|
|
871
|
-
const require = createRequire(import.meta.url);
|
|
872
|
-
let pkgRoot = null;
|
|
873
|
-
try {
|
|
874
|
-
pkgRoot = dirname(require.resolve('@midscene/computer/package.json'));
|
|
875
|
-
} catch {
|
|
876
|
-
const hereDir = dirname(fileURLToPath(import.meta.url));
|
|
877
|
-
for (const candidate of [
|
|
878
|
-
external_node_path_resolve(hereDir, '..'),
|
|
879
|
-
external_node_path_resolve(hereDir, '../..')
|
|
880
|
-
])if (existsSync(external_node_path_resolve(candidate, 'package.json'))) {
|
|
881
|
-
pkgRoot = candidate;
|
|
882
|
-
break;
|
|
883
|
-
}
|
|
884
|
-
}
|
|
885
|
-
if (!pkgRoot) {
|
|
886
|
-
debugDevice(`${helperName}: cannot locate @midscene/computer package root`);
|
|
887
|
-
return null;
|
|
888
|
-
}
|
|
889
|
-
return pkgRoot;
|
|
890
|
-
}
|
|
891
|
-
let phasedScrollBinaryPath;
|
|
892
|
-
function getPhasedScrollBinary() {
|
|
893
|
-
if (void 0 !== phasedScrollBinaryPath) return phasedScrollBinaryPath;
|
|
894
|
-
if ('darwin' !== process.platform) {
|
|
895
|
-
phasedScrollBinaryPath = null;
|
|
896
|
-
return null;
|
|
897
|
-
}
|
|
898
|
-
const pkgRoot = resolvePackageRoot('phased-scroll');
|
|
899
|
-
if (!pkgRoot) {
|
|
900
|
-
phasedScrollBinaryPath = null;
|
|
901
|
-
return null;
|
|
902
|
-
}
|
|
903
|
-
const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/phased-scroll');
|
|
904
|
-
if (!existsSync(binPath)) {
|
|
905
|
-
debugDevice('phased-scroll binary not found at', binPath);
|
|
906
|
-
phasedScrollBinaryPath = null;
|
|
907
|
-
return null;
|
|
908
|
-
}
|
|
909
|
-
try {
|
|
910
|
-
const st = statSync(binPath);
|
|
911
|
-
if ((73 & st.mode) === 0) {
|
|
912
|
-
chmodSync(binPath, 493);
|
|
913
|
-
debugDevice('phased-scroll: restored executable bit on', binPath);
|
|
914
|
-
}
|
|
915
|
-
} catch (err) {
|
|
916
|
-
debugDevice('phased-scroll: chmod self-heal failed', err);
|
|
917
|
-
}
|
|
918
|
-
phasedScrollBinaryPath = binPath;
|
|
919
|
-
return binPath;
|
|
920
|
-
}
|
|
921
|
-
let displayInfoBinaryPath;
|
|
922
|
-
function getDisplayInfoBinary() {
|
|
923
|
-
if (void 0 !== displayInfoBinaryPath) return displayInfoBinaryPath;
|
|
924
|
-
if ('darwin' !== process.platform) {
|
|
925
|
-
displayInfoBinaryPath = null;
|
|
926
|
-
return null;
|
|
927
|
-
}
|
|
928
|
-
const pkgRoot = resolvePackageRoot('display-info');
|
|
929
|
-
if (!pkgRoot) {
|
|
930
|
-
displayInfoBinaryPath = null;
|
|
931
|
-
return null;
|
|
932
|
-
}
|
|
933
|
-
const binPath = external_node_path_resolve(pkgRoot, 'bin/darwin/display-info');
|
|
934
|
-
if (!existsSync(binPath)) {
|
|
935
|
-
debugDevice('display-info binary not found at', binPath);
|
|
936
|
-
displayInfoBinaryPath = null;
|
|
937
|
-
return null;
|
|
938
|
-
}
|
|
939
|
-
displayInfoBinaryPath = binPath;
|
|
940
|
-
return binPath;
|
|
941
|
-
}
|
|
942
|
-
function device_isFiniteNumber(value) {
|
|
943
|
-
return 'number' == typeof value && Number.isFinite(value);
|
|
944
|
-
}
|
|
945
|
-
function isDarwinDisplayGeometry(value) {
|
|
946
|
-
if (!value || 'object' != typeof value) return false;
|
|
947
|
-
const candidate = value;
|
|
948
|
-
return Number.isInteger(candidate.screenIndex) && Number.isInteger(candidate.cgDisplayId) && 'boolean' == typeof candidate.primary && !!candidate.bounds && device_isFiniteNumber(candidate.bounds.x) && device_isFiniteNumber(candidate.bounds.y) && device_isFiniteNumber(candidate.bounds.width) && device_isFiniteNumber(candidate.bounds.height);
|
|
949
|
-
}
|
|
950
|
-
function readDarwinDisplayGeometries() {
|
|
951
|
-
const bin = getDisplayInfoBinary();
|
|
952
|
-
if (!bin) return [];
|
|
953
|
-
try {
|
|
954
|
-
const output = execFileSync(bin, [], {
|
|
955
|
-
encoding: 'utf8',
|
|
956
|
-
stdio: [
|
|
957
|
-
'ignore',
|
|
958
|
-
'pipe',
|
|
959
|
-
'pipe'
|
|
960
|
-
]
|
|
961
|
-
});
|
|
962
|
-
const parsed = JSON.parse(output);
|
|
963
|
-
return Array.isArray(parsed.displays) ? parsed.displays.filter(isDarwinDisplayGeometry) : [];
|
|
964
|
-
} catch (error) {
|
|
965
|
-
debugDevice('display-info helper failed:', error);
|
|
966
|
-
return [];
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
function readDarwinFrontmostApplication() {
|
|
970
|
-
try {
|
|
971
|
-
const output = execFileSync("osascript", [
|
|
972
|
-
'-e',
|
|
973
|
-
'tell application "System Events"\nset frontApp to first application process whose frontmost is true\nreturn (unix id of frontApp as string) & " " & (name of frontApp as string)\nend tell'
|
|
974
|
-
]).toString().trim();
|
|
975
|
-
const [pidText, ...nameParts] = output.split('\t');
|
|
976
|
-
const pid = Number(pidText);
|
|
977
|
-
if (!Number.isInteger(pid) || pid <= 0) return;
|
|
978
|
-
return {
|
|
979
|
-
pid,
|
|
980
|
-
name: nameParts.join('\t')
|
|
981
|
-
};
|
|
982
|
-
} catch (error) {
|
|
983
|
-
debugDevice('Failed to read macOS frontmost application:', error);
|
|
984
|
-
return;
|
|
985
|
-
}
|
|
986
|
-
}
|
|
987
|
-
async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, current, holdDuration, reason) {
|
|
988
|
-
const drift = {
|
|
989
|
-
x: current.x - targetX,
|
|
990
|
-
y: current.y - targetY
|
|
991
|
-
};
|
|
992
|
-
debugComputerInput('tap mouse moved %o', {
|
|
993
|
-
reason,
|
|
994
|
-
target: {
|
|
995
|
-
x: targetX,
|
|
996
|
-
y: targetY
|
|
997
|
-
},
|
|
998
|
-
current,
|
|
999
|
-
drift
|
|
1000
|
-
});
|
|
1001
|
-
await inputDriver.withMouseButton('left', async ()=>{
|
|
1002
|
-
debugComputerInput('tap mouse down %o', {
|
|
1003
|
-
reason
|
|
1004
|
-
});
|
|
1005
|
-
await inputDriver.delay(holdDuration);
|
|
1006
|
-
});
|
|
1007
|
-
debugComputerInput('tap mouse up %o', {
|
|
1008
|
-
reason
|
|
1009
|
-
});
|
|
1010
|
-
}
|
|
1011
|
-
function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
1012
|
-
if (!displays.length) return;
|
|
1013
|
-
const screenIndex = void 0 === displayId || '' === displayId ? 0 : Number(displayId);
|
|
1014
|
-
if (!Number.isInteger(screenIndex) || screenIndex < 0) return void debugDevice('Invalid macOS display id for display geometry:', displayId);
|
|
1015
|
-
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
1016
|
-
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
1017
|
-
}
|
|
1018
|
-
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
1019
|
-
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
1020
|
-
if ('win32' === process.platform) {
|
|
1021
|
-
const geometry = resolveWindowsDisplayGeometryFromList(displayId, windowsDisplays ?? readWindowsDisplayGeometries());
|
|
1022
|
-
if (!geometry) throw new Error(displayId ? `Requested Windows display not found: ${displayId}` : 'No Windows displays were detected');
|
|
1023
|
-
return geometry;
|
|
1024
|
-
}
|
|
1025
|
-
}
|
|
1026
|
-
function mapDisplayLocalPointToGlobal(point, geometry) {
|
|
1027
|
-
if (!geometry) return point;
|
|
1028
|
-
return {
|
|
1029
|
-
x: point.x + geometry.bounds.x,
|
|
1030
|
-
y: point.y + geometry.bounds.y
|
|
1031
|
-
};
|
|
1032
|
-
}
|
|
1033
|
-
let phasedScrollExecWarned = false;
|
|
1034
|
-
function runPhasedScroll(direction, pixels, steps) {
|
|
1035
|
-
const bin = getPhasedScrollBinary();
|
|
1036
|
-
if (!bin) return false;
|
|
1037
|
-
try {
|
|
1038
|
-
const res = spawnSync(bin, [
|
|
1039
|
-
direction,
|
|
1040
|
-
String(Math.max(1, Math.round(pixels))),
|
|
1041
|
-
String(steps)
|
|
1042
|
-
], {
|
|
1043
|
-
stdio: 'ignore'
|
|
1044
|
-
});
|
|
1045
|
-
if (0 === res.status) return true;
|
|
1046
|
-
if (!phasedScrollExecWarned) {
|
|
1047
|
-
phasedScrollExecWarned = true;
|
|
1048
|
-
const hint = null === res.status ? `signal ${res.signal ?? 'unknown'}; the binary may not be executable (npm tarball extraction can drop the +x bit) or may be blocked by quarantine. Try: chmod +x "${bin}"` : 'this usually means Accessibility permission has not been granted to the host process (System Settings → Privacy & Security → Accessibility)';
|
|
1049
|
-
console.warn(`[@midscene/computer] phased-scroll helper failed (exit=${res.status}, signal=${res.signal ?? 'none'}); falling back to keyboard/libnut. ${hint}`);
|
|
1050
|
-
}
|
|
1051
|
-
debugDevice('phased-scroll exited non-zero', res.status, res.signal, res.error);
|
|
1052
|
-
return false;
|
|
1053
|
-
} catch (err) {
|
|
1054
|
-
if (!phasedScrollExecWarned) {
|
|
1055
|
-
phasedScrollExecWarned = true;
|
|
1056
|
-
console.warn(`[@midscene/computer] phased-scroll helper failed to spawn (${err?.message}); falling back to keyboard/libnut.`);
|
|
1057
|
-
}
|
|
1058
|
-
debugDevice('phased-scroll spawn failed', err);
|
|
1059
|
-
return false;
|
|
1060
|
-
}
|
|
1061
|
-
}
|
|
1062
|
-
const KEY_NAME_MAP = {
|
|
1063
|
-
windows: 'win',
|
|
1064
|
-
win: 'win',
|
|
1065
|
-
ctrl: 'control',
|
|
1066
|
-
esc: 'escape',
|
|
1067
|
-
del: 'delete',
|
|
1068
|
-
ins: 'insert',
|
|
1069
|
-
pgup: 'pageup',
|
|
1070
|
-
pgdn: 'pagedown',
|
|
1071
|
-
arrowup: 'up',
|
|
1072
|
-
arrowdown: 'down',
|
|
1073
|
-
arrowleft: 'left',
|
|
1074
|
-
arrowright: 'right',
|
|
1075
|
-
volumedown: 'audio_vol_down',
|
|
1076
|
-
volumeup: 'audio_vol_up',
|
|
1077
|
-
mediavolumedown: 'audio_vol_down',
|
|
1078
|
-
mediavolumeup: 'audio_vol_up',
|
|
1079
|
-
mute: 'audio_mute',
|
|
1080
|
-
mediamute: 'audio_mute',
|
|
1081
|
-
mediaplay: 'audio_play',
|
|
1082
|
-
mediapause: 'audio_pause',
|
|
1083
|
-
mediaplaypause: 'audio_play',
|
|
1084
|
-
mediastop: 'audio_stop',
|
|
1085
|
-
medianexttrack: 'audio_next',
|
|
1086
|
-
mediaprevioustrack: 'audio_prev',
|
|
1087
|
-
medianext: 'audio_next',
|
|
1088
|
-
mediaprev: 'audio_prev'
|
|
1089
|
-
};
|
|
1090
|
-
const PRIMARY_KEY_MAP = {
|
|
1091
|
-
command: 'cmd',
|
|
1092
|
-
cmd: 'cmd',
|
|
1093
|
-
meta: 'meta',
|
|
1094
|
-
control: 'control',
|
|
1095
|
-
ctrl: 'control',
|
|
1096
|
-
shift: 'shift',
|
|
1097
|
-
alt: 'alt',
|
|
1098
|
-
option: 'alt'
|
|
1099
|
-
};
|
|
1100
|
-
function normalizeKeyName(key) {
|
|
1101
|
-
const lowerKey = key.toLowerCase();
|
|
1102
|
-
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1103
|
-
}
|
|
1104
|
-
function normalizePrimaryKey(key) {
|
|
1105
|
-
const lowerKey = key.toLowerCase();
|
|
1106
|
-
if (PRIMARY_KEY_MAP[lowerKey]) return PRIMARY_KEY_MAP[lowerKey];
|
|
1107
|
-
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
1108
|
-
}
|
|
1109
|
-
class ComputerDevice {
|
|
1110
|
-
async moveGlobalPointer(point, context, smooth) {
|
|
1111
|
-
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1112
|
-
const target = {
|
|
1113
|
-
x: Math.round(point.x),
|
|
1114
|
-
y: Math.round(point.y)
|
|
1115
|
-
};
|
|
1116
|
-
if ('win32' === process.platform) {
|
|
1117
|
-
const actual = this.windowsPointerDriver.moveTo(target, {
|
|
1118
|
-
smoothSteps: smooth?.smoothSteps,
|
|
1119
|
-
smoothDelayMs: smooth?.smoothDelay
|
|
1120
|
-
});
|
|
1121
|
-
const drift = windowsPointerDrift(target, actual);
|
|
1122
|
-
if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`${context}: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
|
|
1123
|
-
await this.inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
1124
|
-
return actual;
|
|
1125
|
-
}
|
|
1126
|
-
if (smooth) await this.inputDriver.smoothMoveMouse(target.x, target.y, smooth.smoothSteps, smooth.smoothDelay);
|
|
1127
|
-
else this.inputDriver.moveMouse(target.x, target.y);
|
|
1128
|
-
await this.inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
1129
|
-
return this.inputDriver.getMousePos();
|
|
1130
|
-
}
|
|
1131
|
-
moveDisplayPointer(point, context, smooth) {
|
|
1132
|
-
return this.moveGlobalPointer(this.toGlobalPoint(point), context, smooth);
|
|
1133
|
-
}
|
|
1134
|
-
async performPointerDrag(from, to, duration) {
|
|
1135
|
-
const screenSize = await this.size();
|
|
1136
|
-
const boundedFrom = clampPointerPointToSize(from, screenSize);
|
|
1137
|
-
const boundedTo = clampPointerPointToSize(to, screenSize);
|
|
1138
|
-
await this.moveDisplayPointer(boundedFrom, 'Mouse did not reach the drag start target');
|
|
1139
|
-
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1140
|
-
await this.inputDriver.delay(100);
|
|
1141
|
-
await this.moveDisplayPointer(boundedTo, 'Mouse did not reach the drag end target', {
|
|
1142
|
-
smoothSteps: SMOOTH_MOVE_STEPS_DRAG,
|
|
1143
|
-
smoothDelay: void 0 === duration ? SMOOTH_MOVE_DELAY_DRAG : Math.max(0, Math.round(duration / SMOOTH_MOVE_STEPS_DRAG))
|
|
1144
|
-
});
|
|
1145
|
-
await this.inputDriver.delay(100);
|
|
1146
|
-
});
|
|
1147
|
-
}
|
|
1148
|
-
async focusKeyboardTarget(element, delayMs) {
|
|
1149
|
-
const [x, y] = element.center;
|
|
1150
|
-
if ('darwin' === process.platform) await this.inputPrimitives.pointer.tap({
|
|
1151
|
-
x,
|
|
1152
|
-
y
|
|
1153
|
-
});
|
|
1154
|
-
else {
|
|
1155
|
-
await this.moveDisplayPointer({
|
|
1156
|
-
x,
|
|
1157
|
-
y
|
|
1158
|
-
}, 'Mouse did not reach the keyboard focus target');
|
|
1159
|
-
this.inputDriver.mouseClick('left');
|
|
1160
|
-
}
|
|
1161
|
-
await this.inputDriver.delay(delayMs);
|
|
1162
|
-
}
|
|
1163
|
-
describe() {
|
|
1164
|
-
return this.description || 'Computer Device';
|
|
1165
|
-
}
|
|
1166
|
-
static async listDisplays() {
|
|
1167
|
-
try {
|
|
1168
|
-
if ('win32' === process.platform) return listWindowsDisplays();
|
|
1169
|
-
const displays = await screenshot_desktop.listDisplays();
|
|
1170
|
-
return displays.map((d)=>({
|
|
1171
|
-
id: String(d.id),
|
|
1172
|
-
name: d.name || `Display ${d.id}`,
|
|
1173
|
-
primary: d.primary || false
|
|
1174
|
-
}));
|
|
1175
|
-
} catch (error) {
|
|
1176
|
-
debugDevice(`Failed to list displays: ${error}`);
|
|
1177
|
-
throw new Error(`Failed to list displays: ${error}`);
|
|
1178
|
-
}
|
|
1179
|
-
}
|
|
1180
|
-
async connect() {
|
|
1181
|
-
debugDevice('Connecting to computer device');
|
|
1182
|
-
try {
|
|
1183
|
-
const headless = this.options?.headless ?? 'true' === process.env.MIDSCENE_COMPUTER_HEADLESS_LINUX;
|
|
1184
|
-
if (needsXvfb(headless)) {
|
|
1185
|
-
if (!checkXvfbInstalled()) throw new Error('Xvfb is required for headless mode but not installed. Install: sudo apt-get install xvfb');
|
|
1186
|
-
this.xvfbInstance = await startXvfb({
|
|
1187
|
-
resolution: this.options?.xvfbResolution
|
|
1188
|
-
});
|
|
1189
|
-
if (this.options?.keepXvfbAliveUntilProcessExit) scheduleXvfbStopAfterProcessExit(this.xvfbInstance);
|
|
1190
|
-
process.env.DISPLAY = this.xvfbInstance.display;
|
|
1191
|
-
debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
|
|
1192
|
-
if (!this.options?.keepXvfbAliveUntilProcessExit) {
|
|
1193
|
-
this.xvfbCleanup = ()=>{
|
|
1194
|
-
if (this.xvfbInstance) {
|
|
1195
|
-
this.xvfbInstance.stop();
|
|
1196
|
-
this.xvfbInstance = void 0;
|
|
1197
|
-
}
|
|
1198
|
-
};
|
|
1199
|
-
this.xvfbSignalCleanup = createXvfbSignalCleanup(()=>this.xvfbCleanup?.());
|
|
1200
|
-
process.on('exit', this.xvfbCleanup);
|
|
1201
|
-
process.on('SIGINT', this.xvfbSignalCleanup);
|
|
1202
|
-
process.on('SIGTERM', this.xvfbSignalCleanup);
|
|
1203
|
-
}
|
|
1204
|
-
}
|
|
1205
|
-
device_libnut = await getLibnut();
|
|
1206
|
-
const windowsDisplayGeometries = 'win32' === process.platform ? readWindowsDisplayGeometries() : void 0;
|
|
1207
|
-
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
1208
|
-
const size = await this.size();
|
|
1209
|
-
const displays = windowsDisplayGeometries ? listWindowsDisplays(windowsDisplayGeometries) : await ComputerDevice.listDisplays();
|
|
1210
|
-
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
1211
|
-
this.description = `
|
|
1212
|
-
Type: Computer
|
|
1213
|
-
Platform: ${process.platform}
|
|
1214
|
-
Display: ${this.displayId || 'Primary'}
|
|
1215
|
-
Screen Size: ${size.width}x${size.height}
|
|
1216
|
-
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
1217
|
-
`;
|
|
1218
|
-
debugDevice('Computer device connected', this.description);
|
|
1219
|
-
await this.healthCheck(displays);
|
|
1220
|
-
} catch (error) {
|
|
1221
|
-
if (this.xvfbInstance) {
|
|
1222
|
-
if (!this.options?.keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
1223
|
-
this.xvfbInstance = void 0;
|
|
1224
|
-
}
|
|
1225
|
-
if (this.xvfbCleanup) {
|
|
1226
|
-
process.removeListener('exit', this.xvfbCleanup);
|
|
1227
|
-
this.xvfbCleanup = void 0;
|
|
1228
|
-
}
|
|
1229
|
-
if (this.xvfbSignalCleanup) {
|
|
1230
|
-
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
1231
|
-
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
1232
|
-
this.xvfbSignalCleanup = void 0;
|
|
1233
|
-
}
|
|
1234
|
-
debugDevice(`Failed to connect: ${error}`);
|
|
1235
|
-
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
1236
|
-
}
|
|
1237
|
-
}
|
|
1238
|
-
async healthCheck(displays) {
|
|
1239
|
-
console.log('[HealthCheck] Starting health check...');
|
|
1240
|
-
console.log("[HealthCheck] @midscene/computer v1.17.8-beta-20261106101231.0");
|
|
1241
|
-
console.log('[HealthCheck] Taking screenshot...');
|
|
1242
|
-
const screenshotTimeout = 15000;
|
|
1243
|
-
let timeoutId;
|
|
1244
|
-
const timeoutPromise = new Promise((_, reject)=>{
|
|
1245
|
-
timeoutId = setTimeout(()=>reject(new Error('Screenshot timed out')), screenshotTimeout);
|
|
1246
|
-
});
|
|
1247
|
-
const base64 = await Promise.race([
|
|
1248
|
-
this.screenshotBase64().finally(()=>clearTimeout(timeoutId)),
|
|
1249
|
-
timeoutPromise
|
|
1250
|
-
]);
|
|
1251
|
-
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1252
|
-
console.log('[HealthCheck] Verifying mouse control...');
|
|
1253
|
-
const startPos = 'win32' === process.platform ? this.windowsPointerDriver.getPosition() : this.inputDriver.getMousePos();
|
|
1254
|
-
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1255
|
-
if ('win32' === process.platform) {
|
|
1256
|
-
if (!this.displayGeometry) throw new Error('Windows display geometry is unavailable');
|
|
1257
|
-
const bounds = this.displayGeometry.bounds;
|
|
1258
|
-
const target = {
|
|
1259
|
-
x: Math.round(bounds.x + bounds.width / 2),
|
|
1260
|
-
y: Math.round(bounds.y + bounds.height / 2)
|
|
1261
|
-
};
|
|
1262
|
-
try {
|
|
1263
|
-
const actual = this.windowsPointerDriver.moveTo(target);
|
|
1264
|
-
const drift = windowsPointerDrift(target, actual);
|
|
1265
|
-
if (!windowsPointerIsWithinTolerance(drift)) throw new Error(`Windows screenshot-space pointer verification: expected (${target.x}, ${target.y}), got (${actual.x}, ${actual.y}), drift=(${drift.x}, ${drift.y})`);
|
|
1266
|
-
} finally{
|
|
1267
|
-
this.windowsPointerDriver.moveTo(startPos);
|
|
1268
|
-
}
|
|
1269
|
-
console.log(`[HealthCheck] Mouse verified in screenshot-space display bounds (${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`);
|
|
1270
|
-
} else {
|
|
1271
|
-
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
1272
|
-
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
1273
|
-
const targetX = startPos.x + offsetX;
|
|
1274
|
-
const targetY = startPos.y + offsetY;
|
|
1275
|
-
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
1276
|
-
try {
|
|
1277
|
-
this.inputDriver.moveMouse(targetX, targetY);
|
|
1278
|
-
await sleep(CLICK_SETTLE_DELAY);
|
|
1279
|
-
this.inputDriver.assertMousePosition(targetX, targetY, 'Mouse health check failed');
|
|
1280
|
-
} finally{
|
|
1281
|
-
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
1282
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
1283
|
-
}
|
|
1284
|
-
}
|
|
1285
|
-
if ('win32' === process.platform && !this.isRunningAsAdmin()) {
|
|
1286
|
-
const hint = [
|
|
1287
|
-
'Heads-up: Midscene is not running as Administrator.',
|
|
1288
|
-
'If clicks or key presses have no effect while the cursor still moves to the right position,',
|
|
1289
|
-
`see the Windows permission troubleshooting guide: ${WINDOWS_UIPI_DOC_URL}`
|
|
1290
|
-
].join(' ');
|
|
1291
|
-
warnDevice(`[HealthCheck] ${hint}`);
|
|
1292
|
-
}
|
|
1293
|
-
console.log('[HealthCheck] Listing monitors...');
|
|
1294
|
-
if (displays.length > 0) {
|
|
1295
|
-
console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
|
|
1296
|
-
for (const display of displays){
|
|
1297
|
-
const primaryTag = display.primary ? ' (primary)' : '';
|
|
1298
|
-
console.log(`[HealthCheck] - id=${display.id}, name=${display.name}${primaryTag}`);
|
|
1299
|
-
}
|
|
1300
|
-
} else console.log('[HealthCheck] No monitors detected');
|
|
1301
|
-
console.log('[HealthCheck] Health check passed');
|
|
1302
|
-
}
|
|
1303
|
-
isRunningAsAdmin() {
|
|
1304
|
-
if ('win32' !== process.platform) return false;
|
|
1305
|
-
if (void 0 !== this.adminCheckCache) return this.adminCheckCache;
|
|
1306
|
-
try {
|
|
1307
|
-
execSync('net session', {
|
|
1308
|
-
stdio: 'pipe'
|
|
1309
|
-
});
|
|
1310
|
-
this.adminCheckCache = true;
|
|
1311
|
-
} catch {
|
|
1312
|
-
this.adminCheckCache = false;
|
|
1313
|
-
}
|
|
1314
|
-
return this.adminCheckCache;
|
|
1315
|
-
}
|
|
1316
|
-
async screenshotBase64() {
|
|
1317
|
-
if (this.destroyed) throw new Error('ComputerDevice has been destroyed');
|
|
1318
|
-
debugDevice('Taking screenshot', {
|
|
1319
|
-
displayId: this.displayId
|
|
1320
|
-
});
|
|
1321
|
-
if ('win32' === process.platform) return this.screenshotViaPowershell();
|
|
1322
|
-
const options = {
|
|
1323
|
-
format: 'png'
|
|
1324
|
-
};
|
|
1325
|
-
if (void 0 !== this.displayId) if ('darwin' === process.platform) {
|
|
1326
|
-
const screenIndex = Number(this.displayId);
|
|
1327
|
-
if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
|
|
1328
|
-
} else options.screen = this.displayId;
|
|
1329
|
-
debugDevice('Screenshot options', options);
|
|
1330
|
-
const MAX_ATTEMPTS = 3;
|
|
1331
|
-
const RETRY_DELAY_MS = 300;
|
|
1332
|
-
let lastRawMessage = '';
|
|
1333
|
-
for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
|
|
1334
|
-
const buffer = await captureScreenshot(options);
|
|
1335
|
-
if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
|
|
1336
|
-
return createImgBase64ByFormat('png', buffer.toString('base64'));
|
|
1337
|
-
} catch (error) {
|
|
1338
|
-
lastRawMessage = error instanceof Error ? error.message : String(error);
|
|
1339
|
-
const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
|
|
1340
|
-
const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
|
|
1341
|
-
debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
|
|
1342
|
-
if (!willRetry) break;
|
|
1343
|
-
await sleep(RETRY_DELAY_MS);
|
|
1344
|
-
}
|
|
1345
|
-
if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
|
|
1346
|
-
|
|
1347
|
-
Please follow these steps:
|
|
1348
|
-
1. Open System Settings > Privacy & Security > Screen Recording
|
|
1349
|
-
2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
|
|
1350
|
-
3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
|
|
1351
|
-
|
|
1352
|
-
Original error: ${lastRawMessage}`);
|
|
1353
|
-
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
1354
|
-
}
|
|
1355
|
-
screenshotViaPowershell() {
|
|
1356
|
-
const deviceName = this.displayId ? String(this.displayId) : '';
|
|
1357
|
-
const selectScreen = deviceName ? `$dn = '${escapePowershellSingleQuoted(deviceName)}'
|
|
1358
|
-
$screen = [System.Windows.Forms.Screen]::AllScreens | Where-Object { $_.DeviceName -eq $dn } | Select-Object -First 1
|
|
1359
|
-
if (-not $screen) { throw "Requested display not found: $dn" }` : '$screen = [System.Windows.Forms.Screen]::PrimaryScreen';
|
|
1360
|
-
const script = `
|
|
1361
|
-
Add-Type -AssemblyName System.Windows.Forms, System.Drawing
|
|
1362
|
-
${selectScreen}
|
|
1363
|
-
$b = $screen.Bounds
|
|
1364
|
-
$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height)
|
|
1365
|
-
$g = [System.Drawing.Graphics]::FromImage($bmp)
|
|
1366
|
-
$g.CopyFromScreen($b.X, $b.Y, 0, 0, $bmp.Size)
|
|
1367
|
-
$ms = New-Object System.IO.MemoryStream
|
|
1368
|
-
$bmp.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png)
|
|
1369
|
-
[Console]::Out.Write([Convert]::ToBase64String($ms.ToArray()))
|
|
1370
|
-
$g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
1371
|
-
`.trim();
|
|
1372
|
-
let stdout;
|
|
1373
|
-
try {
|
|
1374
|
-
stdout = runWindowsPhysicalPixelPowershell(script);
|
|
1375
|
-
} catch (error) {
|
|
1376
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1377
|
-
throw new Error(`Failed to take screenshot on Windows: ${message}`);
|
|
1378
|
-
}
|
|
1379
|
-
const body = stdout.trim();
|
|
1380
|
-
if (!body) throw new Error('Failed to take screenshot on Windows: PowerShell returned no image data');
|
|
1381
|
-
return createImgBase64ByFormat('png', body);
|
|
1382
|
-
}
|
|
1383
|
-
async size() {
|
|
1384
|
-
if (this.displayGeometry) return {
|
|
1385
|
-
width: Math.round(this.displayGeometry.bounds.width),
|
|
1386
|
-
height: Math.round(this.displayGeometry.bounds.height)
|
|
1387
|
-
};
|
|
1388
|
-
try {
|
|
1389
|
-
const screenSize = this.inputDriver.getScreenSize();
|
|
1390
|
-
return {
|
|
1391
|
-
width: screenSize.width,
|
|
1392
|
-
height: screenSize.height
|
|
1393
|
-
};
|
|
1394
|
-
} catch (error) {
|
|
1395
|
-
debugDevice(`Failed to get screen size: ${error}`);
|
|
1396
|
-
throw new Error(`Failed to get screen size: ${error}`);
|
|
1397
|
-
}
|
|
1398
|
-
}
|
|
1399
|
-
toGlobalPoint(point) {
|
|
1400
|
-
return mapDisplayLocalPointToGlobal(point, this.displayGeometry);
|
|
1401
|
-
}
|
|
1402
|
-
async typeViaClipboard(text) {
|
|
1403
|
-
debugDevice('Using clipboard to input text', {
|
|
1404
|
-
textLength: text.length,
|
|
1405
|
-
preview: text.substring(0, 20)
|
|
1406
|
-
});
|
|
1407
|
-
const clipboardy = await import("clipboardy");
|
|
1408
|
-
const oldClipboard = await clipboardy.default.read().catch(()=>'');
|
|
1409
|
-
try {
|
|
1410
|
-
await clipboardy.default.write(text);
|
|
1411
|
-
await this.inputDriver.delay(50);
|
|
1412
|
-
if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript('v', [
|
|
1413
|
-
'command'
|
|
1414
|
-
]);
|
|
1415
|
-
else {
|
|
1416
|
-
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
1417
|
-
await this.inputDriver.keyTapWithModifierDelay('v', [
|
|
1418
|
-
modifier
|
|
1419
|
-
], this.options?.keyboardModifierDelay ?? 0);
|
|
1420
|
-
}
|
|
1421
|
-
await this.inputDriver.delay(100);
|
|
1422
|
-
} finally{
|
|
1423
|
-
if (oldClipboard) await clipboardy.default.write(oldClipboard).catch(()=>{
|
|
1424
|
-
debugDevice('Failed to restore clipboard content');
|
|
1425
|
-
});
|
|
1426
|
-
}
|
|
1427
|
-
}
|
|
1428
|
-
async smartTypeString(text, inputOptions) {
|
|
1429
|
-
if (shouldInputSequentially(inputOptions)) return void await this.typeStringWithDelay(text, inputOptions.keyboardTypeDelay ?? 0);
|
|
1430
|
-
await this.typeViaClipboard(text);
|
|
1431
|
-
}
|
|
1432
|
-
async typeStringWithDelay(text, keyboardTypeDelay) {
|
|
1433
|
-
await sendTextSequentially(text.replace(/\r\n?/g, '\n'), {
|
|
1434
|
-
sendCharacter: async (character)=>{
|
|
1435
|
-
const linuxShiftedKey = 'linux' === process.platform ? US_SHIFTED_CHARACTER_KEYS.get(character) : void 0;
|
|
1436
|
-
const pacedShiftedKey = !this.useAppleScript && (this.options?.keyboardModifierDelay ?? 0) > 0 ? resolveShiftedKey(character, this.options?.keyboardLayout) : void 0;
|
|
1437
|
-
if ('\n' === character) this.inputDriver.sendKey('return');
|
|
1438
|
-
else if ('\t' === character) this.inputDriver.sendKey('tab');
|
|
1439
|
-
else if (' ' === character) this.inputDriver.sendKey('space');
|
|
1440
|
-
else if (this.useAppleScript) this.inputDriver.sendKeyViaAppleScript(character);
|
|
1441
|
-
else if (void 0 !== pacedShiftedKey) await this.inputDriver.keyTapWithModifierDelay(pacedShiftedKey, [
|
|
1442
|
-
'shift'
|
|
1443
|
-
], this.options?.keyboardModifierDelay ?? 0);
|
|
1444
|
-
else if (void 0 !== linuxShiftedKey) this.inputDriver.keyTap(linuxShiftedKey, [
|
|
1445
|
-
'shift'
|
|
1446
|
-
]);
|
|
1447
|
-
else this.inputDriver.typeString(character);
|
|
1448
|
-
},
|
|
1449
|
-
wait: (delayMs)=>this.inputDriver.delay(delayMs)
|
|
1450
|
-
}, {
|
|
1451
|
-
delayMs: keyboardTypeDelay
|
|
1452
|
-
});
|
|
1453
|
-
}
|
|
1454
|
-
async selectAllAndDelete() {
|
|
1455
|
-
if (this.useAppleScript) {
|
|
1456
|
-
this.inputDriver.sendKeyViaAppleScript('a', [
|
|
1457
|
-
'command'
|
|
1458
|
-
]);
|
|
1459
|
-
await this.inputDriver.delay(50);
|
|
1460
|
-
this.inputDriver.sendKeyViaAppleScript('backspace', []);
|
|
1461
|
-
return;
|
|
1462
|
-
}
|
|
1463
|
-
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
1464
|
-
await this.inputDriver.keyTapWithModifierDelay('a', [
|
|
1465
|
-
modifier
|
|
1466
|
-
], this.options?.keyboardModifierDelay ?? 0);
|
|
1467
|
-
await this.inputDriver.delay(50);
|
|
1468
|
-
this.inputDriver.keyTap('backspace');
|
|
1469
|
-
}
|
|
1470
|
-
async pressKeyboardShortcut(keyName) {
|
|
1471
|
-
const keys = keyName.split('+');
|
|
1472
|
-
const modifiers = keys.slice(0, -1).map(normalizeKeyName);
|
|
1473
|
-
const key = normalizePrimaryKey(keys[keys.length - 1]);
|
|
1474
|
-
debugDevice('KeyboardPress', {
|
|
1475
|
-
original: keyName,
|
|
1476
|
-
key,
|
|
1477
|
-
modifiers,
|
|
1478
|
-
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
1479
|
-
});
|
|
1480
|
-
if (!this.useAppleScript && modifiers.length > 0 && (this.options?.keyboardModifierDelay ?? 0) > 0) return void await this.inputDriver.keyTapWithModifierDelay(key, modifiers, this.options?.keyboardModifierDelay ?? 0);
|
|
1481
|
-
this.inputDriver.sendKey(key, modifiers);
|
|
1482
|
-
}
|
|
1483
|
-
resolveUntargetedScrollPoint(screenSize) {
|
|
1484
|
-
if ('win32' === process.platform) {
|
|
1485
|
-
const activeWindowRect = this.windowsPointerDriver.getActiveWindowRect();
|
|
1486
|
-
if (activeWindowRect) {
|
|
1487
|
-
const activeWindowCenter = {
|
|
1488
|
-
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
1489
|
-
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
1490
|
-
};
|
|
1491
|
-
const bounds = this.displayGeometry?.bounds;
|
|
1492
|
-
if (!bounds || activeWindowCenter.x >= bounds.x && activeWindowCenter.x < bounds.x + bounds.width && activeWindowCenter.y >= bounds.y && activeWindowCenter.y < bounds.y + bounds.height) return activeWindowCenter;
|
|
1493
|
-
}
|
|
1494
|
-
}
|
|
1495
|
-
return this.toGlobalPoint({
|
|
1496
|
-
x: screenSize.width / 2,
|
|
1497
|
-
y: screenSize.height / 2
|
|
1498
|
-
});
|
|
1499
|
-
}
|
|
1500
|
-
async moveMouseToScrollTarget(param) {
|
|
1501
|
-
if (param.locate) {
|
|
1502
|
-
const element = param.locate;
|
|
1503
|
-
const [x, y] = element.center;
|
|
1504
|
-
await this.moveDisplayPointer({
|
|
1505
|
-
x,
|
|
1506
|
-
y
|
|
1507
|
-
}, 'Mouse did not reach the scroll target');
|
|
1508
|
-
return;
|
|
1509
|
-
}
|
|
1510
|
-
const screenSize = await this.size();
|
|
1511
|
-
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
1512
|
-
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
1513
|
-
await this.moveGlobalPointer(point, 'Mouse did not reach the scroll viewport');
|
|
1514
|
-
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1515
|
-
return screenSize;
|
|
1516
|
-
}
|
|
1517
|
-
async performScroll(param) {
|
|
1518
|
-
let screenSize = await this.moveMouseToScrollTarget(param);
|
|
1519
|
-
const scrollType = param?.scrollType;
|
|
1520
|
-
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
1521
|
-
if (edgeSpec) {
|
|
1522
|
-
if (this.inputDriver.runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
1523
|
-
if (this.useAppleScript) {
|
|
1524
|
-
this.inputDriver.sendKeyViaAppleScript(edgeSpec.key);
|
|
1525
|
-
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
1526
|
-
return;
|
|
1527
|
-
}
|
|
1528
|
-
const [ux, uy] = edgeSpec.libnut;
|
|
1529
|
-
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, LIBNUT_FALLBACK_EDGE_DETENTS, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
1530
|
-
return;
|
|
1531
|
-
}
|
|
1532
|
-
if ('singleAction' === scrollType || !scrollType) {
|
|
1533
|
-
const direction = param?.direction || 'down';
|
|
1534
|
-
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
1535
|
-
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
1536
|
-
let distance = param?.distance ?? void 0;
|
|
1537
|
-
if (!distance) {
|
|
1538
|
-
screenSize ??= await this.size();
|
|
1539
|
-
const base = isHorizontal ? screenSize.width : screenSize.height;
|
|
1540
|
-
distance = Math.max(1, Math.round(base * DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
1541
|
-
}
|
|
1542
|
-
if (isKnownDirection) {
|
|
1543
|
-
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
1544
|
-
if (this.inputDriver.runPhasedScroll(direction, distance, steps)) return void await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
1545
|
-
}
|
|
1546
|
-
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
1547
|
-
if (!screenSize) screenSize = await this.size();
|
|
1548
|
-
const pages = Math.max(1, Math.round(distance / screenSize.height));
|
|
1549
|
-
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
1550
|
-
for(let i = 0; i < pages; i++){
|
|
1551
|
-
this.inputDriver.sendKeyViaAppleScript(key);
|
|
1552
|
-
await this.inputDriver.delay(SCROLL_STEP_DELAY);
|
|
1553
|
-
}
|
|
1554
|
-
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
1555
|
-
return;
|
|
1556
|
-
}
|
|
1557
|
-
const detents = Math.min(LIBNUT_FALLBACK_MAX_DETENTS, Math.max(1, Math.ceil(distance / LIBNUT_FALLBACK_PIXELS_PER_DETENT)));
|
|
1558
|
-
const directionUnit = {
|
|
1559
|
-
up: [
|
|
1560
|
-
0,
|
|
1561
|
-
1
|
|
1562
|
-
],
|
|
1563
|
-
down: [
|
|
1564
|
-
0,
|
|
1565
|
-
-1
|
|
1566
|
-
],
|
|
1567
|
-
left: [
|
|
1568
|
-
-1,
|
|
1569
|
-
0
|
|
1570
|
-
],
|
|
1571
|
-
right: [
|
|
1572
|
-
1,
|
|
1573
|
-
0
|
|
1574
|
-
]
|
|
1575
|
-
};
|
|
1576
|
-
const [ux, uy] = directionUnit[direction] || [
|
|
1577
|
-
0,
|
|
1578
|
-
-1
|
|
1579
|
-
];
|
|
1580
|
-
await this.inputDriver.emitScrollDetents(ux * LIBNUT_FALLBACK_DETENT_AMOUNT, uy * LIBNUT_FALLBACK_DETENT_AMOUNT, detents, LIBNUT_FALLBACK_TICK_DELAY_MS);
|
|
1581
|
-
await this.inputDriver.delay(SCROLL_COMPLETE_DELAY);
|
|
1582
|
-
return;
|
|
1583
|
-
}
|
|
1584
|
-
throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
|
|
1585
|
-
}
|
|
1586
|
-
actionSpace() {
|
|
1587
|
-
const defaultActions = [
|
|
1588
|
-
...defineActionsFromInputPrimitives(this.inputPrimitives, {
|
|
1589
|
-
size: ()=>this.size()
|
|
1590
|
-
})
|
|
1591
|
-
];
|
|
1592
|
-
const platformActions = Object.values(createPlatformActions());
|
|
1593
|
-
const customActions = this.options?.customActions || [];
|
|
1594
|
-
return [
|
|
1595
|
-
...defaultActions,
|
|
1596
|
-
...platformActions,
|
|
1597
|
-
...customActions
|
|
1598
|
-
];
|
|
1599
|
-
}
|
|
1600
|
-
async destroy() {
|
|
1601
|
-
if (this.destroyed) return;
|
|
1602
|
-
this.destroyed = true;
|
|
1603
|
-
this.inputDriver.destroy();
|
|
1604
|
-
const keepXvfbAliveUntilProcessExit = this.options?.keepXvfbAliveUntilProcessExit === true;
|
|
1605
|
-
if (this.xvfbInstance) {
|
|
1606
|
-
if (!keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
1607
|
-
this.xvfbInstance = void 0;
|
|
1608
|
-
}
|
|
1609
|
-
if (this.xvfbCleanup && !keepXvfbAliveUntilProcessExit) {
|
|
1610
|
-
process.removeListener('exit', this.xvfbCleanup);
|
|
1611
|
-
this.xvfbCleanup = void 0;
|
|
1612
|
-
}
|
|
1613
|
-
if (this.xvfbSignalCleanup) {
|
|
1614
|
-
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
1615
|
-
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
1616
|
-
this.xvfbSignalCleanup = void 0;
|
|
1617
|
-
}
|
|
1618
|
-
debugDevice('Computer device destroyed');
|
|
1619
|
-
}
|
|
1620
|
-
async url() {
|
|
1621
|
-
return '';
|
|
1622
|
-
}
|
|
1623
|
-
constructor(options){
|
|
1624
|
-
device_define_property(this, "interfaceType", 'computer');
|
|
1625
|
-
device_define_property(this, "options", void 0);
|
|
1626
|
-
device_define_property(this, "displayId", void 0);
|
|
1627
|
-
device_define_property(this, "displayGeometry", void 0);
|
|
1628
|
-
device_define_property(this, "description", void 0);
|
|
1629
|
-
device_define_property(this, "destroyed", false);
|
|
1630
|
-
device_define_property(this, "xvfbInstance", void 0);
|
|
1631
|
-
device_define_property(this, "xvfbCleanup", void 0);
|
|
1632
|
-
device_define_property(this, "xvfbSignalCleanup", void 0);
|
|
1633
|
-
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
1634
|
-
getLibnut: ()=>device_libnut,
|
|
1635
|
-
useAppleScript: ()=>this.useAppleScript,
|
|
1636
|
-
sendKeyViaAppleScript: sendKeyViaAppleScript,
|
|
1637
|
-
runPhasedScroll,
|
|
1638
|
-
debug: (message)=>debugDevice(message)
|
|
1639
|
-
}));
|
|
1640
|
-
device_define_property(this, "windowsPointerDriver", new WindowsPointerDriver({
|
|
1641
|
-
runPhysicalPixelPowershell: runWindowsPhysicalPixelPowershell
|
|
1642
|
-
}));
|
|
1643
|
-
device_define_property(this, "useAppleScript", void 0);
|
|
1644
|
-
device_define_property(this, "adminCheckCache", void 0);
|
|
1645
|
-
device_define_property(this, "uri", void 0);
|
|
1646
|
-
device_define_property(this, "inputPrimitives", {
|
|
1647
|
-
pointer: {
|
|
1648
|
-
tap: async ({ x, y }, opts)=>{
|
|
1649
|
-
const target = this.toGlobalPoint({
|
|
1650
|
-
x,
|
|
1651
|
-
y
|
|
1652
|
-
});
|
|
1653
|
-
const targetX = Math.round(target.x);
|
|
1654
|
-
const targetY = Math.round(target.y);
|
|
1655
|
-
const holdDuration = Math.max(0, Math.round(opts?.duration ?? CLICK_HOLD_DURATION));
|
|
1656
|
-
debugComputerInput('tap start %o', {
|
|
1657
|
-
local: {
|
|
1658
|
-
x,
|
|
1659
|
-
y
|
|
1660
|
-
},
|
|
1661
|
-
global: {
|
|
1662
|
-
x: targetX,
|
|
1663
|
-
y: targetY
|
|
1664
|
-
},
|
|
1665
|
-
holdDuration,
|
|
1666
|
-
displayId: this.displayId,
|
|
1667
|
-
displayGeometry: this.displayGeometry
|
|
1668
|
-
});
|
|
1669
|
-
const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
|
|
1670
|
-
const current = await this.moveGlobalPointer({
|
|
1671
|
-
x: targetX,
|
|
1672
|
-
y: targetY
|
|
1673
|
-
}, 'Mouse did not reach the tap target', {
|
|
1674
|
-
smoothSteps: SMOOTH_MOVE_STEPS_TAP,
|
|
1675
|
-
smoothDelay: SMOOTH_MOVE_DELAY_TAP
|
|
1676
|
-
});
|
|
1677
|
-
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, current, holdDuration, 'primary');
|
|
1678
|
-
if (frontmostBefore && 'darwin' === process.platform) {
|
|
1679
|
-
await sleep(CLICK_FOCUS_SETTLE_DELAY);
|
|
1680
|
-
const frontmostAfter = readDarwinFrontmostApplication();
|
|
1681
|
-
const focusChanged = !!frontmostAfter && frontmostAfter.pid !== frontmostBefore.pid;
|
|
1682
|
-
debugComputerInput('tap focus check %o', {
|
|
1683
|
-
before: frontmostBefore,
|
|
1684
|
-
after: frontmostAfter,
|
|
1685
|
-
focusChanged
|
|
1686
|
-
});
|
|
1687
|
-
if (focusChanged) {
|
|
1688
|
-
const followUpCurrent = await this.moveGlobalPointer({
|
|
1689
|
-
x: targetX,
|
|
1690
|
-
y: targetY
|
|
1691
|
-
}, 'Mouse did not reach the focus follow-up target');
|
|
1692
|
-
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, followUpCurrent, holdDuration, 'focus-follow-up');
|
|
1693
|
-
}
|
|
1694
|
-
}
|
|
1695
|
-
},
|
|
1696
|
-
doubleClick: async ({ x, y })=>{
|
|
1697
|
-
await this.moveDisplayPointer({
|
|
1698
|
-
x,
|
|
1699
|
-
y
|
|
1700
|
-
}, 'Mouse did not reach the double-click target');
|
|
1701
|
-
this.inputDriver.mouseClick('left', true);
|
|
1702
|
-
},
|
|
1703
|
-
rightClick: async ({ x, y })=>{
|
|
1704
|
-
await this.moveDisplayPointer({
|
|
1705
|
-
x,
|
|
1706
|
-
y
|
|
1707
|
-
}, 'Mouse did not reach the right-click target');
|
|
1708
|
-
this.inputDriver.mouseClick('right');
|
|
1709
|
-
},
|
|
1710
|
-
middleClick: async ({ x, y })=>{
|
|
1711
|
-
await this.moveDisplayPointer({
|
|
1712
|
-
x,
|
|
1713
|
-
y
|
|
1714
|
-
}, 'Mouse did not reach the middle-click target');
|
|
1715
|
-
this.inputDriver.mouseClick('middle');
|
|
1716
|
-
},
|
|
1717
|
-
hover: async ({ x, y })=>{
|
|
1718
|
-
await this.moveDisplayPointer({
|
|
1719
|
-
x,
|
|
1720
|
-
y
|
|
1721
|
-
}, 'Mouse did not reach the hover target', {
|
|
1722
|
-
smoothSteps: SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1723
|
-
smoothDelay: SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
1724
|
-
});
|
|
1725
|
-
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1726
|
-
},
|
|
1727
|
-
dragAndDrop: async (from, to)=>{
|
|
1728
|
-
await this.performPointerDrag(from, to);
|
|
1729
|
-
},
|
|
1730
|
-
swipe: async (from, to, opts)=>{
|
|
1731
|
-
const repeatCount = opts?.repeat ?? 1;
|
|
1732
|
-
for(let index = 0; index < repeatCount; index++)await this.performPointerDrag(from, to, opts?.duration);
|
|
1733
|
-
}
|
|
1734
|
-
},
|
|
1735
|
-
keyboard: {
|
|
1736
|
-
typeText: async (value, opts)=>{
|
|
1737
|
-
const resolvedInputOptions = resolveTextInputOptions(opts, this.options);
|
|
1738
|
-
const element = opts?.target;
|
|
1739
|
-
if (element) {
|
|
1740
|
-
await this.focusKeyboardTarget(element, INPUT_FOCUS_DELAY);
|
|
1741
|
-
if (opts?.replace !== false) {
|
|
1742
|
-
await this.selectAllAndDelete();
|
|
1743
|
-
await this.inputDriver.delay(INPUT_CLEAR_DELAY);
|
|
1744
|
-
}
|
|
1745
|
-
}
|
|
1746
|
-
await this.smartTypeString(value, resolvedInputOptions);
|
|
1747
|
-
},
|
|
1748
|
-
keyboardPress: async (keyName, opts)=>{
|
|
1749
|
-
const target = opts?.target;
|
|
1750
|
-
if (target) await this.focusKeyboardTarget(target, 50);
|
|
1751
|
-
await this.pressKeyboardShortcut(keyName);
|
|
1752
|
-
},
|
|
1753
|
-
clearInput: async (target)=>{
|
|
1754
|
-
if (target) await this.focusKeyboardTarget(target, 100);
|
|
1755
|
-
await this.selectAllAndDelete();
|
|
1756
|
-
await this.inputDriver.delay(50);
|
|
1757
|
-
}
|
|
1758
|
-
},
|
|
1759
|
-
scroll: {
|
|
1760
|
-
scroll: async (param)=>{
|
|
1761
|
-
await this.performScroll(param);
|
|
1762
|
-
}
|
|
1763
|
-
}
|
|
1764
|
-
});
|
|
1765
|
-
if (options?.keyboardModifierDelay !== void 0 && (!Number.isFinite(options.keyboardModifierDelay) || options.keyboardModifierDelay < 0)) throw new Error('keyboardModifierDelay must be a finite non-negative number');
|
|
1766
|
-
if (options?.keyboardLayout !== void 0 && 'en-US' !== options.keyboardLayout) throw new Error('keyboardLayout must be "en-US" when specified');
|
|
1767
|
-
this.options = options;
|
|
1768
|
-
this.displayId = options?.displayId;
|
|
1769
|
-
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
1770
|
-
}
|
|
1771
|
-
}
|
|
1772
|
-
function createPlatformActions() {
|
|
1773
|
-
return {
|
|
1774
|
-
ListDisplays: defineAction({
|
|
1775
|
-
name: 'ListDisplays',
|
|
1776
|
-
supportedBySystemOne: true,
|
|
1777
|
-
description: 'List all available displays/monitors',
|
|
1778
|
-
call: async ()=>await ComputerDevice.listDisplays()
|
|
1779
|
-
})
|
|
1780
|
-
};
|
|
1781
|
-
}
|
|
1782
|
-
function normalizeRdpHost(host) {
|
|
1783
|
-
const trimmed = host.trim();
|
|
1784
|
-
if (trimmed.length >= 2 && trimmed.startsWith('[') && trimmed.endsWith(']') && trimmed.includes(':')) return trimmed.slice(1, -1);
|
|
1785
|
-
return trimmed;
|
|
1786
|
-
}
|
|
1787
|
-
function formatRdpHost(host) {
|
|
1788
|
-
const normalizedHost = normalizeRdpHost(host);
|
|
1789
|
-
return normalizedHost.includes(':') ? `[${normalizedHost}]` : normalizedHost;
|
|
1790
|
-
}
|
|
1791
|
-
function formatRdpServerAddress(host, port) {
|
|
1792
|
-
return `${formatRdpHost(host)}:${port}`;
|
|
1793
|
-
}
|
|
1794
|
-
function normalizeRdpConnectionConfig(config) {
|
|
1795
|
-
return {
|
|
1796
|
-
...config,
|
|
1797
|
-
host: normalizeRdpHost(config.host),
|
|
1798
|
-
...config.localAddress ? {
|
|
1799
|
-
localAddress: config.localAddress.trim()
|
|
1800
|
-
} : {}
|
|
1801
|
-
};
|
|
1802
|
-
}
|
|
1803
|
-
const platformBinaryMap = {
|
|
1804
|
-
darwin: {
|
|
1805
|
-
directory: 'darwin',
|
|
1806
|
-
fileName: 'rdp-helper'
|
|
1807
|
-
},
|
|
1808
|
-
linux: {
|
|
1809
|
-
directory: 'linux',
|
|
1810
|
-
fileName: 'rdp-helper'
|
|
1811
|
-
},
|
|
1812
|
-
win32: {
|
|
1813
|
-
directory: 'win32',
|
|
1814
|
-
fileName: 'rdp-helper.exe'
|
|
1815
|
-
}
|
|
1816
|
-
};
|
|
1817
|
-
function getPlatformBinary(platform) {
|
|
1818
|
-
if (platform in platformBinaryMap) return platformBinaryMap[platform];
|
|
1819
|
-
}
|
|
1820
|
-
function currentDirname() {
|
|
1821
|
-
if ('undefined' != typeof __dirname) return __dirname;
|
|
1822
|
-
return dirname(fileURLToPath(import.meta.url));
|
|
1823
|
-
}
|
|
1824
|
-
function getRdpHelperBinaryPath() {
|
|
1825
|
-
const platformBinary = getPlatformBinary(process.platform);
|
|
1826
|
-
if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
|
|
1827
|
-
const hereDir = currentDirname();
|
|
1828
|
-
const candidateRoots = [
|
|
1829
|
-
external_node_path_resolve(hereDir, '../..'),
|
|
1830
|
-
external_node_path_resolve(hereDir, '../../..')
|
|
1831
|
-
];
|
|
1832
|
-
for (const root of candidateRoots){
|
|
1833
|
-
const binaryPath = external_node_path_resolve(root, 'bin', platformBinary.directory, platformBinary.fileName);
|
|
1834
|
-
if (existsSync(binaryPath)) return binaryPath;
|
|
1835
|
-
}
|
|
1836
|
-
throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
|
|
1837
|
-
}
|
|
1838
|
-
function backend_client_define_property(obj, key, value) {
|
|
1839
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
1840
|
-
value: value,
|
|
1841
|
-
enumerable: true,
|
|
1842
|
-
configurable: true,
|
|
1843
|
-
writable: true
|
|
1844
|
-
});
|
|
1845
|
-
else obj[key] = value;
|
|
1846
|
-
return obj;
|
|
1847
|
-
}
|
|
1848
|
-
const debug = getDebug('rdp:backend');
|
|
1849
|
-
const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
|
|
1850
|
-
const HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS = 50;
|
|
1851
|
-
const MAX_STDERR_CHARS = 16384;
|
|
1852
|
-
const notImplementedError = ()=>new Error('RDP backend transport is not implemented yet. Inject a custom backend into RDPDevice or implement a real helper transport.');
|
|
1853
|
-
class UnsupportedRDPBackendClient {
|
|
1854
|
-
async connect(_config) {
|
|
1855
|
-
debug('connect called on unsupported backend');
|
|
1856
|
-
throw notImplementedError();
|
|
1857
|
-
}
|
|
1858
|
-
async disconnect() {
|
|
1859
|
-
debug('disconnect called on unsupported backend');
|
|
1860
|
-
throw notImplementedError();
|
|
1861
|
-
}
|
|
1862
|
-
async screenshotBase64() {
|
|
1863
|
-
debug('screenshotBase64 called on unsupported backend');
|
|
1864
|
-
throw notImplementedError();
|
|
1865
|
-
}
|
|
1866
|
-
async size() {
|
|
1867
|
-
debug('size called on unsupported backend');
|
|
1868
|
-
throw notImplementedError();
|
|
1869
|
-
}
|
|
1870
|
-
async mouseMove(_x, _y) {
|
|
1871
|
-
debug('mouseMove called on unsupported backend');
|
|
1872
|
-
throw notImplementedError();
|
|
1873
|
-
}
|
|
1874
|
-
async mouseButton(_button, _action) {
|
|
1875
|
-
debug('mouseButton called on unsupported backend');
|
|
1876
|
-
throw notImplementedError();
|
|
1877
|
-
}
|
|
1878
|
-
async wheel(_direction, _amount, _x, _y) {
|
|
1879
|
-
debug('wheel called on unsupported backend');
|
|
1880
|
-
throw notImplementedError();
|
|
1881
|
-
}
|
|
1882
|
-
async keyPress(_keyName) {
|
|
1883
|
-
debug('keyPress called on unsupported backend');
|
|
1884
|
-
throw notImplementedError();
|
|
1885
|
-
}
|
|
1886
|
-
async typeText(_text) {
|
|
1887
|
-
debug('typeText called on unsupported backend');
|
|
1888
|
-
throw notImplementedError();
|
|
1889
|
-
}
|
|
1890
|
-
async clearInput() {
|
|
1891
|
-
debug('clearInput called on unsupported backend');
|
|
1892
|
-
throw notImplementedError();
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
class HelperProcessRDPBackendClient {
|
|
1896
|
-
async connect(config) {
|
|
1897
|
-
this.fatalHelperError = void 0;
|
|
1898
|
-
const response = await this.send({
|
|
1899
|
-
type: 'connect',
|
|
1900
|
-
config: normalizeRdpConnectionConfig(config)
|
|
1901
|
-
});
|
|
1902
|
-
if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
|
|
1903
|
-
this.connected = true;
|
|
1904
|
-
this.fatalHelperError = void 0;
|
|
1905
|
-
return response.info;
|
|
1906
|
-
}
|
|
1907
|
-
async disconnect() {
|
|
1908
|
-
const child = this.child;
|
|
1909
|
-
if (!child) return;
|
|
1910
|
-
let disconnectError;
|
|
1911
|
-
if (this.connected && null === child.exitCode) try {
|
|
1912
|
-
const response = await this.send({
|
|
1913
|
-
type: 'disconnect'
|
|
1914
|
-
});
|
|
1915
|
-
this.expectOk(response, 'disconnect');
|
|
1916
|
-
} catch (error) {
|
|
1917
|
-
disconnectError = error instanceof Error ? error : new Error(String(error));
|
|
1918
|
-
}
|
|
1919
|
-
this.connected = false;
|
|
1920
|
-
this.fatalHelperError = void 0;
|
|
1921
|
-
await this.shutdownHelper();
|
|
1922
|
-
if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
|
|
1923
|
-
}
|
|
1924
|
-
async screenshotBase64() {
|
|
1925
|
-
const response = await this.send({
|
|
1926
|
-
type: 'screenshot'
|
|
1927
|
-
});
|
|
1928
|
-
if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
|
|
1929
|
-
return response.base64;
|
|
1930
|
-
}
|
|
1931
|
-
async size() {
|
|
1932
|
-
const response = await this.send({
|
|
1933
|
-
type: 'size'
|
|
1934
|
-
});
|
|
1935
|
-
if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
|
|
1936
|
-
return response.size;
|
|
1937
|
-
}
|
|
1938
|
-
async mouseMove(x, y) {
|
|
1939
|
-
const response = await this.send({
|
|
1940
|
-
type: 'mouseMove',
|
|
1941
|
-
x,
|
|
1942
|
-
y
|
|
1943
|
-
});
|
|
1944
|
-
this.expectOk(response, 'mouseMove');
|
|
1945
|
-
}
|
|
1946
|
-
async mouseButton(button, action) {
|
|
1947
|
-
const response = await this.send({
|
|
1948
|
-
type: 'mouseButton',
|
|
1949
|
-
button,
|
|
1950
|
-
action
|
|
1951
|
-
});
|
|
1952
|
-
this.expectOk(response, 'mouseButton');
|
|
1953
|
-
}
|
|
1954
|
-
async wheel(direction, amount, x, y) {
|
|
1955
|
-
const response = await this.send({
|
|
1956
|
-
type: 'wheel',
|
|
1957
|
-
direction,
|
|
1958
|
-
amount,
|
|
1959
|
-
x,
|
|
1960
|
-
y
|
|
1961
|
-
});
|
|
1962
|
-
this.expectOk(response, 'wheel');
|
|
1963
|
-
}
|
|
1964
|
-
async keyPress(keyName) {
|
|
1965
|
-
const response = await this.send({
|
|
1966
|
-
type: 'keyPress',
|
|
1967
|
-
keyName
|
|
1968
|
-
});
|
|
1969
|
-
this.expectOk(response, 'keyPress');
|
|
1970
|
-
}
|
|
1971
|
-
async typeText(text) {
|
|
1972
|
-
const response = await this.send({
|
|
1973
|
-
type: 'typeText',
|
|
1974
|
-
text
|
|
1975
|
-
});
|
|
1976
|
-
this.expectOk(response, 'typeText');
|
|
1977
|
-
}
|
|
1978
|
-
async clearInput() {
|
|
1979
|
-
const response = await this.send({
|
|
1980
|
-
type: 'clearInput'
|
|
1981
|
-
});
|
|
1982
|
-
this.expectOk(response, 'clearInput');
|
|
1983
|
-
}
|
|
1984
|
-
ensureHelperStarted() {
|
|
1985
|
-
if (this.child && null === this.child.exitCode) return this.child;
|
|
1986
|
-
const helperPath = this.resolveHelperPath();
|
|
1987
|
-
debug('starting rdp helper', {
|
|
1988
|
-
helperPath
|
|
1989
|
-
});
|
|
1990
|
-
const child = this.spawnFn(helperPath, [], {
|
|
1991
|
-
stdio: [
|
|
1992
|
-
'pipe',
|
|
1993
|
-
'pipe',
|
|
1994
|
-
'pipe'
|
|
1995
|
-
]
|
|
1996
|
-
});
|
|
1997
|
-
child.stdout.setEncoding('utf8');
|
|
1998
|
-
child.stderr.setEncoding('utf8');
|
|
1999
|
-
this.child = child;
|
|
2000
|
-
const diagnostics = {
|
|
2001
|
-
path: helperPath,
|
|
2002
|
-
pid: child.pid,
|
|
2003
|
-
stderrBuffer: ''
|
|
2004
|
-
};
|
|
2005
|
-
this.helperDiagnostics.set(child, diagnostics);
|
|
2006
|
-
debug('started rdp helper', {
|
|
2007
|
-
helperPath,
|
|
2008
|
-
pid: child.pid
|
|
2009
|
-
});
|
|
2010
|
-
this.stdoutReader = createInterface({
|
|
2011
|
-
input: child.stdout,
|
|
2012
|
-
crlfDelay: 1 / 0
|
|
2013
|
-
});
|
|
2014
|
-
this.stdoutReader.on('line', (line)=>{
|
|
2015
|
-
this.handleStdoutLine(child, line, diagnostics);
|
|
2016
|
-
});
|
|
2017
|
-
child.stderr.on('data', (chunk)=>{
|
|
2018
|
-
this.captureStderrChunk(chunk, diagnostics);
|
|
2019
|
-
});
|
|
2020
|
-
child.stdin.on('error', (error)=>{
|
|
2021
|
-
this.handleHelperStreamError(child, 'stdin', error, diagnostics);
|
|
2022
|
-
});
|
|
2023
|
-
child.stdout.on('error', (error)=>{
|
|
2024
|
-
this.handleHelperStreamError(child, 'stdout', error, diagnostics);
|
|
2025
|
-
});
|
|
2026
|
-
child.stderr.on('error', (error)=>{
|
|
2027
|
-
this.handleHelperStreamError(child, 'stderr', error, diagnostics);
|
|
2028
|
-
});
|
|
2029
|
-
child.on('exit', (code, signal)=>{
|
|
2030
|
-
diagnostics.exit = {
|
|
2031
|
-
code,
|
|
2032
|
-
signal
|
|
2033
|
-
};
|
|
2034
|
-
debug('rdp helper exited', {
|
|
2035
|
-
helperPath: diagnostics.path,
|
|
2036
|
-
pid: diagnostics.pid,
|
|
2037
|
-
code,
|
|
2038
|
-
signal
|
|
2039
|
-
});
|
|
2040
|
-
const helperError = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`, void 0, diagnostics);
|
|
2041
|
-
this.rejectPendingForChild(child, helperError);
|
|
2042
|
-
if (this.child === child) {
|
|
2043
|
-
this.connected = false;
|
|
2044
|
-
this.fatalHelperError = helperError;
|
|
2045
|
-
this.disposeReaders();
|
|
2046
|
-
this.child = void 0;
|
|
2047
|
-
}
|
|
2048
|
-
});
|
|
2049
|
-
child.on('error', (error)=>{
|
|
2050
|
-
const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`, void 0, diagnostics);
|
|
2051
|
-
this.rejectPendingForChild(child, helperError);
|
|
2052
|
-
if (this.child === child) {
|
|
2053
|
-
this.connected = false;
|
|
2054
|
-
this.fatalHelperError = helperError;
|
|
2055
|
-
this.disposeReaders();
|
|
2056
|
-
this.child = void 0;
|
|
2057
|
-
}
|
|
2058
|
-
});
|
|
2059
|
-
return child;
|
|
2060
|
-
}
|
|
2061
|
-
handleHelperStreamError(child, streamName, error, diagnostics) {
|
|
2062
|
-
const nodeError = error;
|
|
2063
|
-
const helperError = this.createHelperError(`RDP helper ${streamName} stream error: ${error.message}`, nodeError.code, diagnostics);
|
|
2064
|
-
this.rejectPendingForChild(child, helperError);
|
|
2065
|
-
if (this.child !== child) return;
|
|
2066
|
-
this.connected = false;
|
|
2067
|
-
this.fatalHelperError = helperError;
|
|
2068
|
-
if (null === child.exitCode) child.kill('SIGTERM');
|
|
2069
|
-
this.child = void 0;
|
|
2070
|
-
}
|
|
2071
|
-
handleStdoutLine(child, line, diagnostics) {
|
|
2072
|
-
if (!line.trim()) return;
|
|
2073
|
-
let parsed;
|
|
2074
|
-
try {
|
|
2075
|
-
parsed = JSON.parse(line);
|
|
2076
|
-
} catch (error) {
|
|
2077
|
-
const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`, void 0, diagnostics);
|
|
2078
|
-
this.rejectPendingForChild(child, protocolError);
|
|
2079
|
-
if (this.child === child) this.shutdownHelper(protocolError);
|
|
2080
|
-
return;
|
|
2081
|
-
}
|
|
2082
|
-
const pending = this.pending.get(parsed.id);
|
|
2083
|
-
if (!pending) return void debug('dropping response for unknown request id', parsed);
|
|
2084
|
-
this.pending.delete(parsed.id);
|
|
2085
|
-
if (parsed.ok) return void pending.resolve(parsed.payload);
|
|
2086
|
-
pending.reject(this.createHelperError(parsed.error.message, parsed.error.code, diagnostics));
|
|
2087
|
-
}
|
|
2088
|
-
captureStderrChunk(chunk, diagnostics) {
|
|
2089
|
-
const text = chunk.toString();
|
|
2090
|
-
if (!text.trim()) return;
|
|
2091
|
-
debug('rdp helper stderr', {
|
|
2092
|
-
helperPath: diagnostics.path,
|
|
2093
|
-
pid: diagnostics.pid,
|
|
2094
|
-
stderr: text.trim()
|
|
2095
|
-
});
|
|
2096
|
-
diagnostics.stderrBuffer += text;
|
|
2097
|
-
if (diagnostics.stderrBuffer.length > MAX_STDERR_CHARS) diagnostics.stderrBuffer = diagnostics.stderrBuffer.slice(-MAX_STDERR_CHARS);
|
|
2098
|
-
}
|
|
2099
|
-
async send(payload) {
|
|
2100
|
-
if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
|
|
2101
|
-
const child = this.ensureHelperStarted();
|
|
2102
|
-
if (null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
|
|
2103
|
-
const id = `req-${++this.nextRequestId}`;
|
|
2104
|
-
const diagnostics = this.helperDiagnostics.get(child);
|
|
2105
|
-
const request = {
|
|
2106
|
-
id,
|
|
2107
|
-
payload
|
|
2108
|
-
};
|
|
2109
|
-
return new Promise((resolve, reject)=>{
|
|
2110
|
-
this.pending.set(id, {
|
|
2111
|
-
resolve,
|
|
2112
|
-
reject,
|
|
2113
|
-
child
|
|
2114
|
-
});
|
|
2115
|
-
child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
|
|
2116
|
-
if (!error) return;
|
|
2117
|
-
this.pending.delete(id);
|
|
2118
|
-
const nodeError = error;
|
|
2119
|
-
const timer = setTimeout(()=>{
|
|
2120
|
-
reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`, nodeError.code, diagnostics));
|
|
2121
|
-
}, HELPER_WRITE_ERROR_DIAGNOSTIC_DELAY_MS);
|
|
2122
|
-
timer.unref?.();
|
|
2123
|
-
});
|
|
2124
|
-
});
|
|
2125
|
-
}
|
|
2126
|
-
expectOk(response, actionName) {
|
|
2127
|
-
if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
|
|
2128
|
-
}
|
|
2129
|
-
rejectPending(error) {
|
|
2130
|
-
for (const { reject } of this.pending.values())reject(error);
|
|
2131
|
-
this.pending.clear();
|
|
2132
|
-
}
|
|
2133
|
-
rejectPendingForChild(child, error) {
|
|
2134
|
-
for (const [id, pending] of this.pending)if (pending.child === child) {
|
|
2135
|
-
pending.reject(error);
|
|
2136
|
-
this.pending.delete(id);
|
|
2137
|
-
}
|
|
2138
|
-
}
|
|
2139
|
-
createHelperError(message, code, diagnostics) {
|
|
2140
|
-
const diagnosticParts = [
|
|
2141
|
-
diagnostics?.path ? `path=${diagnostics.path}` : void 0,
|
|
2142
|
-
'number' == typeof diagnostics?.pid ? `pid=${diagnostics.pid}` : void 0,
|
|
2143
|
-
diagnostics?.exit ? `exitCode=${diagnostics.exit.code}, signal=${diagnostics.exit.signal}` : void 0
|
|
2144
|
-
].filter(Boolean);
|
|
2145
|
-
const diagnosticsSuffix = diagnosticParts.length > 0 ? `\nHelper diagnostics: ${diagnosticParts.join(', ')}` : '';
|
|
2146
|
-
const stderrSummary = diagnostics?.stderrBuffer.trim();
|
|
2147
|
-
const stderrSuffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
|
|
2148
|
-
const error = new Error(`${message}${diagnosticsSuffix}${stderrSuffix}`);
|
|
2149
|
-
if (code) error.name = code;
|
|
2150
|
-
return error;
|
|
2151
|
-
}
|
|
2152
|
-
disposeReaders() {
|
|
2153
|
-
this.stdoutReader?.close();
|
|
2154
|
-
this.stdoutReader = void 0;
|
|
2155
|
-
}
|
|
2156
|
-
async shutdownHelper(rootError) {
|
|
2157
|
-
const child = this.child;
|
|
2158
|
-
this.child = void 0;
|
|
2159
|
-
this.disposeReaders();
|
|
2160
|
-
if (!child) return;
|
|
2161
|
-
this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
|
|
2162
|
-
if (null !== child.exitCode) return;
|
|
2163
|
-
child.stdin.end();
|
|
2164
|
-
const exited = Promise.race([
|
|
2165
|
-
once(child, 'exit'),
|
|
2166
|
-
new Promise((resolve)=>{
|
|
2167
|
-
setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
|
|
2168
|
-
})
|
|
2169
|
-
]);
|
|
2170
|
-
const result = await exited;
|
|
2171
|
-
if ('timeout' !== result) return;
|
|
2172
|
-
child.kill('SIGTERM');
|
|
2173
|
-
const terminated = Promise.race([
|
|
2174
|
-
once(child, 'exit'),
|
|
2175
|
-
new Promise((resolve)=>{
|
|
2176
|
-
setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
|
|
2177
|
-
})
|
|
2178
|
-
]);
|
|
2179
|
-
const terminateResult = await terminated;
|
|
2180
|
-
if ('timeout' !== terminateResult) return;
|
|
2181
|
-
child.kill('SIGKILL');
|
|
2182
|
-
await once(child, 'exit');
|
|
2183
|
-
}
|
|
2184
|
-
constructor(options){
|
|
2185
|
-
backend_client_define_property(this, "spawnFn", void 0);
|
|
2186
|
-
backend_client_define_property(this, "resolveHelperPath", void 0);
|
|
2187
|
-
backend_client_define_property(this, "child", void 0);
|
|
2188
|
-
backend_client_define_property(this, "stdoutReader", void 0);
|
|
2189
|
-
backend_client_define_property(this, "pending", new Map());
|
|
2190
|
-
backend_client_define_property(this, "helperDiagnostics", new WeakMap());
|
|
2191
|
-
backend_client_define_property(this, "nextRequestId", 0);
|
|
2192
|
-
backend_client_define_property(this, "connected", false);
|
|
2193
|
-
backend_client_define_property(this, "fatalHelperError", void 0);
|
|
2194
|
-
this.spawnFn = options?.spawnFn || spawn;
|
|
2195
|
-
const overridePath = options?.helperPath;
|
|
2196
|
-
this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
|
|
2197
|
-
}
|
|
2198
|
-
}
|
|
2199
|
-
function createDefaultRDPBackendClient() {
|
|
2200
|
-
return new HelperProcessRDPBackendClient();
|
|
2201
|
-
}
|
|
2202
|
-
function rdp_device_define_property(obj, key, value) {
|
|
2203
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
2204
|
-
value: value,
|
|
2205
|
-
enumerable: true,
|
|
2206
|
-
configurable: true,
|
|
2207
|
-
writable: true
|
|
2208
|
-
});
|
|
2209
|
-
else obj[key] = value;
|
|
2210
|
-
return obj;
|
|
2211
|
-
}
|
|
2212
|
-
const device_debug = getDebug('rdp:device');
|
|
2213
|
-
const device_SMOOTH_MOVE_STEPS_TAP = 8;
|
|
2214
|
-
const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
2215
|
-
const device_SMOOTH_MOVE_STEPS_DRAG = 12;
|
|
2216
|
-
const device_SMOOTH_MOVE_DELAY_TAP = 8;
|
|
2217
|
-
const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
2218
|
-
const device_SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
2219
|
-
const device_MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
2220
|
-
const device_CLICK_HOLD_DURATION = 50;
|
|
2221
|
-
const DRAG_HOLD_DURATION = 100;
|
|
2222
|
-
const device_INPUT_FOCUS_DELAY = 300;
|
|
2223
|
-
const device_INPUT_CLEAR_DELAY = 150;
|
|
2224
|
-
const device_SCROLL_STEP_DELAY = 100;
|
|
2225
|
-
const device_SCROLL_COMPLETE_DELAY = 500;
|
|
2226
|
-
const DEFAULT_SCROLL_DISTANCE = 480;
|
|
2227
|
-
const device_DEFAULT_SCROLL_VIEWPORT_RATIO = 0.7;
|
|
2228
|
-
const device_EDGE_SCROLL_STEPS = 10;
|
|
2229
|
-
const DEFAULT_SCROLL_STEP_AMOUNT = 120;
|
|
2230
|
-
class RDPDevice {
|
|
2231
|
-
async typeText(value, inputOptions) {
|
|
2232
|
-
if (!shouldInputSequentially(inputOptions)) return void await this.backend.typeText(value);
|
|
2233
|
-
await sendTextSequentially(value, {
|
|
2234
|
-
sendCharacter: (character)=>this.backend.typeText(character),
|
|
2235
|
-
wait: sleep
|
|
2236
|
-
}, {
|
|
2237
|
-
delayMs: inputOptions.keyboardTypeDelay
|
|
2238
|
-
});
|
|
2239
|
-
}
|
|
2240
|
-
describe() {
|
|
2241
|
-
const port = this.connectionConfig.port || 3389;
|
|
2242
|
-
const server = formatRdpServerAddress(this.connectionConfig.host, port);
|
|
2243
|
-
const username = this.connectionConfig.username ? ` as ${this.connectionConfig.username}` : '';
|
|
2244
|
-
const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
|
|
2245
|
-
return `RDP Device ${server}${username}${session}`;
|
|
2246
|
-
}
|
|
2247
|
-
async connect() {
|
|
2248
|
-
this.throwIfDestroyed();
|
|
2249
|
-
device_debug('connecting to rdp backend', {
|
|
2250
|
-
host: this.connectionConfig.host,
|
|
2251
|
-
port: this.connectionConfig.port,
|
|
2252
|
-
username: this.connectionConfig.username
|
|
2253
|
-
});
|
|
2254
|
-
this.connectionInfo = await this.backend.connect(this.connectionConfig);
|
|
2255
|
-
this.cursorPosition = [
|
|
2256
|
-
Math.round(this.connectionInfo.size.width / 2),
|
|
2257
|
-
Math.round(this.connectionInfo.size.height / 2)
|
|
2258
|
-
];
|
|
2259
|
-
}
|
|
2260
|
-
async screenshotBase64() {
|
|
2261
|
-
this.assertConnected();
|
|
2262
|
-
return this.backend.screenshotBase64();
|
|
2263
|
-
}
|
|
2264
|
-
async size() {
|
|
2265
|
-
this.assertConnected();
|
|
2266
|
-
return this.backend.size();
|
|
2267
|
-
}
|
|
2268
|
-
async destroy() {
|
|
2269
|
-
if (this.destroyed) return;
|
|
2270
|
-
this.destroyed = true;
|
|
2271
|
-
this.connectionInfo = void 0;
|
|
2272
|
-
this.cursorPosition = void 0;
|
|
2273
|
-
await this.backend.disconnect();
|
|
2274
|
-
}
|
|
2275
|
-
actionSpace() {
|
|
2276
|
-
const defaultActions = [
|
|
2277
|
-
...defineActionsFromInputPrimitives(this.inputPrimitives, {
|
|
2278
|
-
size: ()=>this.size()
|
|
2279
|
-
}),
|
|
2280
|
-
defineAction({
|
|
2281
|
-
name: 'ListDisplays',
|
|
2282
|
-
supportedBySystemOne: true,
|
|
2283
|
-
description: 'List all available displays/monitors',
|
|
2284
|
-
call: async ()=>{
|
|
2285
|
-
this.assertConnected();
|
|
2286
|
-
const size = await this.size();
|
|
2287
|
-
const server = this.connectionInfo?.server || formatRdpServerAddress(this.connectionConfig.host, this.connectionConfig.port || 3389);
|
|
2288
|
-
return [
|
|
2289
|
-
{
|
|
2290
|
-
id: this.connectionInfo?.sessionId || this.connectionConfig.host,
|
|
2291
|
-
name: `RDP ${server} (${size.width}x${size.height})`,
|
|
2292
|
-
primary: true
|
|
2293
|
-
}
|
|
2294
|
-
];
|
|
2295
|
-
}
|
|
2296
|
-
})
|
|
2297
|
-
];
|
|
2298
|
-
return [
|
|
2299
|
-
...defaultActions,
|
|
2300
|
-
...this.customActions
|
|
2301
|
-
];
|
|
2302
|
-
}
|
|
2303
|
-
assertConnected() {
|
|
2304
|
-
this.throwIfDestroyed();
|
|
2305
|
-
if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
|
|
2306
|
-
}
|
|
2307
|
-
throwIfDestroyed() {
|
|
2308
|
-
if (this.destroyed) throw new Error('RDPDevice has been destroyed');
|
|
2309
|
-
}
|
|
2310
|
-
async moveToElement(element, options) {
|
|
2311
|
-
this.assertConnected();
|
|
2312
|
-
const targetX = Math.round(element.center[0]);
|
|
2313
|
-
const targetY = Math.round(element.center[1]);
|
|
2314
|
-
await this.movePointer(targetX, targetY, options);
|
|
2315
|
-
}
|
|
2316
|
-
async clearInput() {
|
|
2317
|
-
if (this.backend.clearInput) return void await this.backend.clearInput();
|
|
2318
|
-
await this.backend.keyPress('Control+A');
|
|
2319
|
-
await this.backend.keyPress('Backspace');
|
|
2320
|
-
}
|
|
2321
|
-
edgeScrollDirection(scrollType) {
|
|
2322
|
-
switch(scrollType){
|
|
2323
|
-
case 'scrollToTop':
|
|
2324
|
-
return 'up';
|
|
2325
|
-
case 'scrollToBottom':
|
|
2326
|
-
return 'down';
|
|
2327
|
-
case 'scrollToLeft':
|
|
2328
|
-
return 'left';
|
|
2329
|
-
case 'scrollToRight':
|
|
2330
|
-
return 'right';
|
|
2331
|
-
case 'singleAction':
|
|
2332
|
-
return 'down';
|
|
2333
|
-
default:
|
|
2334
|
-
throw new Error(`Unsupported scroll type: ${scrollType}`);
|
|
2335
|
-
}
|
|
2336
|
-
}
|
|
2337
|
-
defaultScrollDistance(direction) {
|
|
2338
|
-
const size = this.connectionInfo?.size;
|
|
2339
|
-
if (!size) return DEFAULT_SCROLL_DISTANCE;
|
|
2340
|
-
const isHorizontal = 'left' === direction || 'right' === direction;
|
|
2341
|
-
const base = isHorizontal ? size.width : size.height;
|
|
2342
|
-
return Math.max(1, Math.round(base * device_DEFAULT_SCROLL_VIEWPORT_RATIO));
|
|
2343
|
-
}
|
|
2344
|
-
async movePointer(targetX, targetY, options) {
|
|
2345
|
-
this.assertConnected();
|
|
2346
|
-
const start = this.cursorPosition || [
|
|
2347
|
-
targetX,
|
|
2348
|
-
targetY
|
|
2349
|
-
];
|
|
2350
|
-
const steps = Math.max(1, options?.steps || 1);
|
|
2351
|
-
const stepDelayMs = options?.stepDelayMs || 0;
|
|
2352
|
-
for(let step = 1; step <= steps; step++){
|
|
2353
|
-
const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
|
|
2354
|
-
const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
|
|
2355
|
-
await this.backend.mouseMove(x, y);
|
|
2356
|
-
this.cursorPosition = [
|
|
2357
|
-
x,
|
|
2358
|
-
y
|
|
2359
|
-
];
|
|
2360
|
-
if (stepDelayMs > 0 && step < steps) await sleep(stepDelayMs);
|
|
2361
|
-
}
|
|
2362
|
-
if (options?.settleDelayMs) await sleep(options.settleDelayMs);
|
|
2363
|
-
}
|
|
2364
|
-
async performPointerDrag(from, to, duration) {
|
|
2365
|
-
const screenSize = await this.size();
|
|
2366
|
-
const boundedFrom = clampPointerPointToSize(from, screenSize);
|
|
2367
|
-
const boundedTo = clampPointerPointToSize(to, screenSize);
|
|
2368
|
-
await this.movePointer(Math.round(boundedFrom.x), Math.round(boundedFrom.y), {
|
|
2369
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
2370
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
2371
|
-
});
|
|
2372
|
-
await this.backend.mouseButton('left', 'down');
|
|
2373
|
-
try {
|
|
2374
|
-
await sleep(DRAG_HOLD_DURATION);
|
|
2375
|
-
await this.movePointer(Math.round(boundedTo.x), Math.round(boundedTo.y), {
|
|
2376
|
-
steps: device_SMOOTH_MOVE_STEPS_DRAG,
|
|
2377
|
-
stepDelayMs: void 0 === duration ? device_SMOOTH_MOVE_DELAY_DRAG : Math.max(0, Math.round(duration / device_SMOOTH_MOVE_STEPS_DRAG))
|
|
2378
|
-
});
|
|
2379
|
-
await sleep(DRAG_HOLD_DURATION);
|
|
2380
|
-
} finally{
|
|
2381
|
-
await this.backend.mouseButton('left', 'up');
|
|
2382
|
-
}
|
|
2383
|
-
}
|
|
2384
|
-
async performWheel(direction, amount, x, y) {
|
|
2385
|
-
let remaining = Math.abs(amount);
|
|
2386
|
-
if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
|
|
2387
|
-
while(remaining > 0){
|
|
2388
|
-
const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
|
|
2389
|
-
await this.backend.wheel(direction, chunk, x, y);
|
|
2390
|
-
remaining -= chunk;
|
|
2391
|
-
if (remaining > 0) await sleep(device_SCROLL_STEP_DELAY);
|
|
2392
|
-
}
|
|
2393
|
-
}
|
|
2394
|
-
constructor(options){
|
|
2395
|
-
rdp_device_define_property(this, "interfaceType", 'rdp');
|
|
2396
|
-
rdp_device_define_property(this, "connectionConfig", void 0);
|
|
2397
|
-
rdp_device_define_property(this, "inputOptions", void 0);
|
|
2398
|
-
rdp_device_define_property(this, "customActions", void 0);
|
|
2399
|
-
rdp_device_define_property(this, "backend", void 0);
|
|
2400
|
-
rdp_device_define_property(this, "connectionInfo", void 0);
|
|
2401
|
-
rdp_device_define_property(this, "destroyed", false);
|
|
2402
|
-
rdp_device_define_property(this, "cursorPosition", void 0);
|
|
2403
|
-
rdp_device_define_property(this, "uri", void 0);
|
|
2404
|
-
rdp_device_define_property(this, "inputPrimitives", {
|
|
2405
|
-
pointer: {
|
|
2406
|
-
tap: async ({ x, y })=>{
|
|
2407
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
2408
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
2409
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
2410
|
-
});
|
|
2411
|
-
await this.backend.mouseButton('left', 'down');
|
|
2412
|
-
await sleep(device_CLICK_HOLD_DURATION);
|
|
2413
|
-
await this.backend.mouseButton('left', 'up');
|
|
2414
|
-
},
|
|
2415
|
-
doubleClick: async ({ x, y })=>{
|
|
2416
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
2417
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
2418
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
2419
|
-
});
|
|
2420
|
-
await this.backend.mouseButton('left', 'doubleClick');
|
|
2421
|
-
},
|
|
2422
|
-
rightClick: async ({ x, y })=>{
|
|
2423
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
2424
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
2425
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
2426
|
-
});
|
|
2427
|
-
await this.backend.mouseButton('right', 'click');
|
|
2428
|
-
},
|
|
2429
|
-
middleClick: async ({ x, y })=>{
|
|
2430
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
2431
|
-
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
2432
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
2433
|
-
});
|
|
2434
|
-
await this.backend.mouseButton('middle', 'click');
|
|
2435
|
-
},
|
|
2436
|
-
hover: async ({ x, y })=>{
|
|
2437
|
-
await this.movePointer(Math.round(x), Math.round(y), {
|
|
2438
|
-
steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
2439
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
|
|
2440
|
-
settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
|
|
2441
|
-
});
|
|
2442
|
-
},
|
|
2443
|
-
dragAndDrop: async (from, to)=>{
|
|
2444
|
-
await this.performPointerDrag(from, to);
|
|
2445
|
-
},
|
|
2446
|
-
swipe: async (from, to, opts)=>{
|
|
2447
|
-
const repeatCount = opts?.repeat ?? 1;
|
|
2448
|
-
for(let index = 0; index < repeatCount; index++)await this.performPointerDrag(from, to, opts?.duration);
|
|
2449
|
-
}
|
|
2450
|
-
},
|
|
2451
|
-
keyboard: {
|
|
2452
|
-
typeText: async (value, opts)=>{
|
|
2453
|
-
this.assertConnected();
|
|
2454
|
-
const resolvedInputOptions = resolveTextInputOptions(opts, this.inputOptions);
|
|
2455
|
-
const target = opts?.target;
|
|
2456
|
-
if (target) {
|
|
2457
|
-
await this.inputPrimitives.pointer.tap({
|
|
2458
|
-
x: target.center[0],
|
|
2459
|
-
y: target.center[1]
|
|
2460
|
-
});
|
|
2461
|
-
await sleep(device_INPUT_FOCUS_DELAY);
|
|
2462
|
-
}
|
|
2463
|
-
if (opts?.replace !== false) {
|
|
2464
|
-
await this.clearInput();
|
|
2465
|
-
await sleep(device_INPUT_CLEAR_DELAY);
|
|
2466
|
-
}
|
|
2467
|
-
if (opts?.focusOnly || !value) return;
|
|
2468
|
-
await this.typeText(value, resolvedInputOptions);
|
|
2469
|
-
},
|
|
2470
|
-
clearInput: async (target)=>{
|
|
2471
|
-
this.assertConnected();
|
|
2472
|
-
const element = target;
|
|
2473
|
-
if (element) {
|
|
2474
|
-
await this.inputPrimitives.pointer.tap({
|
|
2475
|
-
x: element.center[0],
|
|
2476
|
-
y: element.center[1]
|
|
2477
|
-
});
|
|
2478
|
-
await sleep(device_INPUT_FOCUS_DELAY);
|
|
2479
|
-
}
|
|
2480
|
-
await this.clearInput();
|
|
2481
|
-
await sleep(device_INPUT_CLEAR_DELAY);
|
|
2482
|
-
},
|
|
2483
|
-
keyboardPress: async (keyName, opts)=>{
|
|
2484
|
-
this.assertConnected();
|
|
2485
|
-
const target = opts?.target;
|
|
2486
|
-
if (target) await this.inputPrimitives.pointer.tap({
|
|
2487
|
-
x: target.center[0],
|
|
2488
|
-
y: target.center[1]
|
|
2489
|
-
});
|
|
2490
|
-
await this.backend.keyPress(keyName);
|
|
2491
|
-
}
|
|
2492
|
-
},
|
|
2493
|
-
scroll: {
|
|
2494
|
-
scroll: async (param)=>{
|
|
2495
|
-
this.assertConnected();
|
|
2496
|
-
const target = param.locate;
|
|
2497
|
-
if (target) await this.moveToElement(target, {
|
|
2498
|
-
steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
2499
|
-
stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
2500
|
-
});
|
|
2501
|
-
if (param.scrollType && 'singleAction' !== param.scrollType) {
|
|
2502
|
-
const direction = this.edgeScrollDirection(param.scrollType);
|
|
2503
|
-
for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
|
|
2504
|
-
await sleep(device_SCROLL_COMPLETE_DELAY);
|
|
2505
|
-
return;
|
|
2506
|
-
}
|
|
2507
|
-
await this.performWheel(param.direction || 'down', param.distance || this.defaultScrollDistance(param.direction || 'down'), target?.center[0], target?.center[1]);
|
|
2508
|
-
await sleep(device_SCROLL_COMPLETE_DELAY);
|
|
2509
|
-
}
|
|
2510
|
-
}
|
|
2511
|
-
});
|
|
2512
|
-
const { backend, customActions, inputStrategy, keyboardTypeDelay, ...connectionConfig } = options;
|
|
2513
|
-
this.connectionConfig = {
|
|
2514
|
-
port: 3389,
|
|
2515
|
-
securityProtocol: 'auto',
|
|
2516
|
-
ignoreCertificate: false,
|
|
2517
|
-
...normalizeRdpConnectionConfig(connectionConfig)
|
|
2518
|
-
};
|
|
2519
|
-
this.inputOptions = {
|
|
2520
|
-
inputStrategy,
|
|
2521
|
-
keyboardTypeDelay
|
|
2522
|
-
};
|
|
2523
|
-
this.customActions = customActions ?? [];
|
|
2524
|
-
this.backend = backend || createDefaultRDPBackendClient();
|
|
2525
|
-
}
|
|
2526
|
-
}
|
|
2527
|
-
class ComputerAgent extends Agent {
|
|
2528
|
-
}
|
|
2529
|
-
function createLocalComputerDevice(opts) {
|
|
2530
|
-
return new ComputerDevice({
|
|
2531
|
-
displayId: opts?.displayId,
|
|
2532
|
-
customActions: opts?.customActions,
|
|
2533
|
-
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2534
|
-
inputStrategy: opts?.inputStrategy,
|
|
2535
|
-
keyboardDriver: opts?.keyboardDriver,
|
|
2536
|
-
keyboardModifierDelay: opts?.keyboardModifierDelay,
|
|
2537
|
-
keyboardLayout: opts?.keyboardLayout,
|
|
2538
|
-
headless: opts?.headless,
|
|
2539
|
-
xvfbResolution: opts?.xvfbResolution,
|
|
2540
|
-
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
2541
|
-
});
|
|
2542
|
-
}
|
|
2543
|
-
function createRDPComputerDevice(opts) {
|
|
2544
|
-
return new RDPDevice({
|
|
2545
|
-
host: opts.host,
|
|
2546
|
-
port: opts.port,
|
|
2547
|
-
username: opts.username,
|
|
2548
|
-
password: opts.password,
|
|
2549
|
-
domain: opts.domain,
|
|
2550
|
-
localAddress: opts.localAddress,
|
|
2551
|
-
adminSession: opts.adminSession,
|
|
2552
|
-
ignoreCertificate: opts.ignoreCertificate,
|
|
2553
|
-
securityProtocol: opts.securityProtocol,
|
|
2554
|
-
desktopWidth: opts.desktopWidth,
|
|
2555
|
-
desktopHeight: opts.desktopHeight,
|
|
2556
|
-
backend: opts.backend,
|
|
2557
|
-
customActions: opts.customActions,
|
|
2558
|
-
keyboardTypeDelay: opts.keyboardTypeDelay,
|
|
2559
|
-
inputStrategy: opts.inputStrategy
|
|
2560
|
-
});
|
|
2561
|
-
}
|
|
2562
|
-
async function agentForComputer(opts) {
|
|
2563
|
-
const device = createLocalComputerDevice(opts);
|
|
2564
|
-
await device.connect();
|
|
2565
|
-
return new ComputerAgent(device, opts);
|
|
2566
|
-
}
|
|
2567
|
-
const agentFromComputer = agentForComputer;
|
|
2568
|
-
async function agentForRDPComputer(opts) {
|
|
2569
|
-
const device = createRDPComputerDevice(opts);
|
|
2570
|
-
await device.connect();
|
|
2571
|
-
return new ComputerAgent(device, opts);
|
|
2572
|
-
}
|
|
2573
|
-
function agent_tools_define_property(obj, key, value) {
|
|
2574
|
-
if (key in obj) Object.defineProperty(obj, key, {
|
|
2575
|
-
value: value,
|
|
2576
|
-
enumerable: true,
|
|
2577
|
-
configurable: true,
|
|
2578
|
-
writable: true
|
|
2579
|
-
});
|
|
2580
|
-
else obj[key] = value;
|
|
2581
|
-
return obj;
|
|
2582
|
-
}
|
|
2583
|
-
const agent_tools_debug = getDebug('agent-tools:computer');
|
|
2584
|
-
const RDP_SECURITY_PROTOCOLS = [
|
|
2585
|
-
'auto',
|
|
2586
|
-
'tls',
|
|
2587
|
-
'nla',
|
|
2588
|
-
'rdp'
|
|
2589
|
-
];
|
|
2590
|
-
const computerInitArgShape = {
|
|
2591
|
-
displayId: z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
|
|
2592
|
-
headless: z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
2593
|
-
keyboardTypeDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds between keystrokes. In "legacy" mode, positive values enable key-by-key input; zero or omitted uses clipboard input.'),
|
|
2594
|
-
inputStrategy: z["enum"](inputStrategies).optional().describe('Text input strategy. "legacy" (default) preserves current Computer behavior, "sequential" sends one Unicode code point at a time, and "bulk" uses one backend input operation. "bulk" requires keyboardTypeDelay to be omitted or set to 0.'),
|
|
2595
|
-
keyboardModifierDelay: z.number().finite().nonnegative().optional().describe('Finite non-negative delay in milliseconds after each modifier transition and the main key for local libnut keyboard events. Positive values pace shortcuts and the implicit Shift used by uppercase letters during sequential input. Layout-dependent punctuation also requires keyboardLayout. Ignored in RDP mode and by the macOS AppleScript driver.'),
|
|
2596
|
-
keyboardLayout: z["enum"]([
|
|
2597
|
-
'en-US'
|
|
2598
|
-
]).optional().describe('Keyboard layout used to resolve layout-dependent shifted punctuation to base keys during sequential local libnut input. Currently only "en-US" is supported. Ignored in RDP mode and by the macOS AppleScript driver.'),
|
|
2599
|
-
host: z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
2600
|
-
port: z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
2601
|
-
username: z.string().optional().describe('RDP username. Requires host.'),
|
|
2602
|
-
password: z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
|
|
2603
|
-
domain: z.string().optional().describe('RDP domain. Requires host.'),
|
|
2604
|
-
localAddress: z.string().optional().describe('Local source IP address for the RDP TCP connection. Requires host.'),
|
|
2605
|
-
adminSession: z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
|
|
2606
|
-
ignoreCertificate: z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
|
|
2607
|
-
securityProtocol: z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
|
|
2608
|
-
desktopWidth: z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
|
|
2609
|
-
desktopHeight: z.number().optional().describe('Remote desktop height in pixels. Requires host.'),
|
|
2610
|
-
...agentBehaviorInitArgShape
|
|
2611
|
-
};
|
|
2612
|
-
function adaptComputerInitArgs(extracted) {
|
|
2613
|
-
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
2614
|
-
if (extracted.host) {
|
|
2615
|
-
const { displayId: _d, headless: _h, keyboardModifierDelay: _s, keyboardLayout: _l, ...rdpFields } = extracted;
|
|
2616
|
-
const host = normalizeRdpHost(extracted.host);
|
|
2617
|
-
return {
|
|
2618
|
-
mode: 'rdp',
|
|
2619
|
-
...rdpFields,
|
|
2620
|
-
host
|
|
2621
|
-
};
|
|
2622
|
-
}
|
|
2623
|
-
return {
|
|
2624
|
-
mode: 'local',
|
|
2625
|
-
displayId: extracted.displayId,
|
|
2626
|
-
headless: extracted.headless,
|
|
2627
|
-
keyboardTypeDelay: extracted.keyboardTypeDelay,
|
|
2628
|
-
inputStrategy: extracted.inputStrategy,
|
|
2629
|
-
keyboardModifierDelay: extracted.keyboardModifierDelay,
|
|
2630
|
-
keyboardLayout: extracted.keyboardLayout,
|
|
2631
|
-
...extractAgentBehaviorInitArgs(extracted) ?? {}
|
|
2632
|
-
};
|
|
2633
|
-
}
|
|
2634
|
-
function describeConnectTarget(opts) {
|
|
2635
|
-
if (opts?.mode === 'rdp') {
|
|
2636
|
-
const target = opts.port ? formatRdpServerAddress(opts.host, opts.port) : formatRdpHost(opts.host);
|
|
2637
|
-
const userSuffix = opts.username ? ` as ${opts.username}` : '';
|
|
2638
|
-
return ` via RDP (${target}${userSuffix})`;
|
|
2639
|
-
}
|
|
2640
|
-
if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
|
|
2641
|
-
return ' (Primary display)';
|
|
2642
|
-
}
|
|
2643
|
-
function getCliReportSessionTarget(opts) {
|
|
2644
|
-
if (opts?.mode === 'rdp') return `rdp:${formatRdpHost(opts.host)}`;
|
|
2645
|
-
if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
|
|
2646
|
-
return 'primary';
|
|
2647
|
-
}
|
|
2648
|
-
class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
2649
|
-
getCliReportSessionName() {
|
|
2650
|
-
return 'midscene-computer';
|
|
2651
|
-
}
|
|
2652
|
-
createTemporaryDevice() {
|
|
2653
|
-
return new ComputerDevice({});
|
|
2654
|
-
}
|
|
2655
|
-
async ensureAgent(opts) {
|
|
2656
|
-
const nextSignature = getAgentInitArgsSignature(opts);
|
|
2657
|
-
if (this.agent && shouldRebuildAgentForInitArgs(this.lastInitArgsSignature, nextSignature)) {
|
|
2658
|
-
try {
|
|
2659
|
-
await this.agent.destroy?.();
|
|
2660
|
-
} catch (error) {
|
|
2661
|
-
agent_tools_debug('Failed to destroy agent during cleanup:', error);
|
|
2662
|
-
}
|
|
2663
|
-
this.agent = void 0;
|
|
2664
|
-
}
|
|
2665
|
-
if (this.agent) return this.agent;
|
|
2666
|
-
const reportOptions = this.readCliReportAgentOptions();
|
|
2667
|
-
if (opts?.mode === 'rdp') {
|
|
2668
|
-
agent_tools_debug('Creating RDP Computer agent for host:', opts.host);
|
|
2669
|
-
const { mode: _mode, ...rdpFields } = opts;
|
|
2670
|
-
const agent = await agentForRDPComputer({
|
|
2671
|
-
...rdpFields,
|
|
2672
|
-
...reportOptions ?? {}
|
|
2673
|
-
});
|
|
2674
|
-
this.agent = agent;
|
|
2675
|
-
this.lastInitArgsSignature = nextSignature;
|
|
2676
|
-
return agent;
|
|
2677
|
-
}
|
|
2678
|
-
const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
|
|
2679
|
-
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
2680
|
-
const keyboardTypeDelay = opts?.keyboardTypeDelay;
|
|
2681
|
-
const inputStrategy = opts?.inputStrategy;
|
|
2682
|
-
const keyboardModifierDelay = opts?.mode === 'local' ? opts.keyboardModifierDelay : void 0;
|
|
2683
|
-
const keyboardLayout = opts?.mode === 'local' ? opts.keyboardLayout : void 0;
|
|
2684
|
-
agent_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
2685
|
-
const agentOpts = {
|
|
2686
|
-
...displayId ? {
|
|
2687
|
-
displayId
|
|
2688
|
-
} : {},
|
|
2689
|
-
...void 0 !== headless ? {
|
|
2690
|
-
headless
|
|
2691
|
-
} : {},
|
|
2692
|
-
...void 0 !== keyboardTypeDelay ? {
|
|
2693
|
-
keyboardTypeDelay
|
|
2694
|
-
} : {},
|
|
2695
|
-
...void 0 !== inputStrategy ? {
|
|
2696
|
-
inputStrategy
|
|
2697
|
-
} : {},
|
|
2698
|
-
...void 0 !== keyboardModifierDelay ? {
|
|
2699
|
-
keyboardModifierDelay
|
|
2700
|
-
} : {},
|
|
2701
|
-
...void 0 !== keyboardLayout ? {
|
|
2702
|
-
keyboardLayout
|
|
2703
|
-
} : {},
|
|
2704
|
-
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2705
|
-
keepXvfbAliveUntilProcessExit: true
|
|
2706
|
-
} : {},
|
|
2707
|
-
...extractAgentBehaviorInitArgs(opts) ?? {},
|
|
2708
|
-
...reportOptions ?? {}
|
|
2709
|
-
};
|
|
2710
|
-
const agent = await agentFromComputer(Object.keys(agentOpts).length > 0 ? agentOpts : void 0);
|
|
2711
|
-
this.agent = agent;
|
|
2712
|
-
this.lastInitArgsSignature = nextSignature;
|
|
2713
|
-
return agent;
|
|
2714
|
-
}
|
|
2715
|
-
preparePlatformTools() {
|
|
2716
|
-
return [
|
|
2717
|
-
{
|
|
2718
|
-
name: 'computer_connect',
|
|
2719
|
-
description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/localAddress/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
|
|
2720
|
-
schema: this.getAgentInitArgSchema(),
|
|
2721
|
-
cli: this.getAgentInitArgCliMetadata(),
|
|
2722
|
-
handler: async (args)=>{
|
|
2723
|
-
const initArgs = this.extractAgentInitParam(args);
|
|
2724
|
-
const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
|
|
2725
|
-
this.commitCliReportSession(reportSession);
|
|
2726
|
-
if (this.agent) {
|
|
2727
|
-
try {
|
|
2728
|
-
await this.agent.destroy?.();
|
|
2729
|
-
} catch (error) {
|
|
2730
|
-
agent_tools_debug('Failed to destroy agent during connect:', error);
|
|
2731
|
-
}
|
|
2732
|
-
this.agent = void 0;
|
|
2733
|
-
this.lastInitArgsSignature = void 0;
|
|
2734
|
-
}
|
|
2735
|
-
const agent = await this.ensureAgent(initArgs);
|
|
2736
|
-
const screenshot = await agent.interface.screenshotBase64();
|
|
2737
|
-
return {
|
|
2738
|
-
content: [
|
|
2739
|
-
{
|
|
2740
|
-
type: 'text',
|
|
2741
|
-
text: `Connected to computer${describeConnectTarget(initArgs)}`
|
|
2742
|
-
},
|
|
2743
|
-
...this.buildScreenshotContent(screenshot)
|
|
2744
|
-
]
|
|
2745
|
-
};
|
|
2746
|
-
}
|
|
2747
|
-
},
|
|
2748
|
-
{
|
|
2749
|
-
name: 'computer_disconnect',
|
|
2750
|
-
description: 'Disconnect from computer and release resources',
|
|
2751
|
-
schema: {},
|
|
2752
|
-
handler: this.createDisconnectHandler('computer')
|
|
2753
|
-
},
|
|
2754
|
-
{
|
|
2755
|
-
name: 'computer_list_displays',
|
|
2756
|
-
description: 'List all available displays/monitors',
|
|
2757
|
-
schema: {},
|
|
2758
|
-
handler: async ()=>{
|
|
2759
|
-
const displays = await ComputerDevice.listDisplays();
|
|
2760
|
-
return {
|
|
2761
|
-
content: [
|
|
2762
|
-
{
|
|
2763
|
-
type: 'text',
|
|
2764
|
-
text: `Available displays:\n${displays.map((d)=>`- ${d.name} (ID: ${d.id})${d.primary ? ' [PRIMARY]' : ''}`).join('\n')}`
|
|
2765
|
-
}
|
|
2766
|
-
]
|
|
2767
|
-
};
|
|
2768
|
-
}
|
|
2769
|
-
}
|
|
2770
|
-
];
|
|
2771
|
-
}
|
|
2772
|
-
constructor(options = {}){
|
|
2773
|
-
super(), agent_tools_define_property(this, "options", void 0), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", void 0), this.options = options, this.initArgSpec = {
|
|
2774
|
-
namespace: 'computer',
|
|
2775
|
-
shape: computerInitArgShape,
|
|
2776
|
-
cli: {
|
|
2777
|
-
preferBareKeys: true
|
|
2778
|
-
},
|
|
2779
|
-
adapt: (extracted)=>adaptComputerInitArgs(extracted)
|
|
2780
|
-
};
|
|
2781
|
-
}
|
|
2782
|
-
}
|
|
1
|
+
import { createRequire, ComputerDevice } from "./430.mjs";
|
|
2783
2
|
function version() {
|
|
2784
|
-
const currentVersion = "1.17.8
|
|
3
|
+
const currentVersion = "1.17.8";
|
|
2785
4
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2786
5
|
return currentVersion;
|
|
2787
6
|
}
|
|
@@ -2893,4 +112,6 @@ async function checkComputerEnvironment() {
|
|
|
2893
112
|
async function getConnectedDisplays() {
|
|
2894
113
|
return ComputerDevice.listDisplays();
|
|
2895
114
|
}
|
|
2896
|
-
export { ComputerAgent, ComputerDevice, ComputerMidsceneTools, HelperProcessRDPBackendClient, RDPDevice, UnsupportedRDPBackendClient, agentForComputer, agentForRDPComputer, agentFromComputer,
|
|
115
|
+
export { ComputerAgent, ComputerDevice, ComputerMidsceneTools, HelperProcessRDPBackendClient, RDPDevice, UnsupportedRDPBackendClient, agentForComputer, agentForRDPComputer, agentFromComputer, checkXvfbInstalled, createDefaultRDPBackendClient, needsXvfb } from "./430.mjs";
|
|
116
|
+
export { overrideAIConfig } from "@midscene/shared/env";
|
|
117
|
+
export { checkAccessibilityPermission, checkComputerEnvironment, checkScreenRecordingPermission, getConnectedDisplays, version };
|