@midscene/computer 1.12.2-beta-20260828072235.0 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +152 -283
- package/dist/es/index.mjs +149 -282
- package/dist/lib/cli.js +152 -283
- package/dist/lib/index.js +149 -282
- package/dist/types/index.d.ts +20 -6
- package/package.json +3 -3
package/dist/es/cli.mjs
CHANGED
|
@@ -19,57 +19,6 @@ import node_assert from "node:assert";
|
|
|
19
19
|
import { hasActiveCliInterruptWaiter } from "@midscene/shared/cli/interrupt";
|
|
20
20
|
import { once } from "node:events";
|
|
21
21
|
import { createInterface } from "node:readline";
|
|
22
|
-
const MOUSE_COORDINATE_TOLERANCE_PX = 5;
|
|
23
|
-
const MIN_CALIBRATION_AXIS_DELTA = 20;
|
|
24
|
-
const MIN_VALID_CALIBRATION_SCALE = 0.1;
|
|
25
|
-
const MAX_VALID_CALIBRATION_SCALE = 10;
|
|
26
|
-
function assertValidMouseCalibrationBounds(bounds) {
|
|
27
|
-
if (!Number.isFinite(bounds.x) || !Number.isFinite(bounds.y) || !Number.isFinite(bounds.width) || !Number.isFinite(bounds.height) || bounds.width <= 0 || bounds.height <= 0) throw new Error(`Mouse calibration bounds are invalid: (${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`);
|
|
28
|
-
}
|
|
29
|
-
function mouseCalibrationPoint(bounds, ratio) {
|
|
30
|
-
return {
|
|
31
|
-
x: Math.round(bounds.x + bounds.width * ratio),
|
|
32
|
-
y: Math.round(bounds.y + bounds.height * ratio)
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
|
-
function calculateMouseCoordinateCalibration(firstInput, firstActual, secondInput, secondActual) {
|
|
36
|
-
const inputDeltaX = secondInput.x - firstInput.x;
|
|
37
|
-
const inputDeltaY = secondInput.y - firstInput.y;
|
|
38
|
-
if (Math.abs(inputDeltaX) < MIN_CALIBRATION_AXIS_DELTA || Math.abs(inputDeltaY) < MIN_CALIBRATION_AXIS_DELTA) throw new Error(`Mouse calibration points are too close: delta=(${inputDeltaX}, ${inputDeltaY})`);
|
|
39
|
-
const scaleX = (secondActual.x - firstActual.x) / inputDeltaX;
|
|
40
|
-
const scaleY = (secondActual.y - firstActual.y) / inputDeltaY;
|
|
41
|
-
if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY) || scaleX < MIN_VALID_CALIBRATION_SCALE || scaleY < MIN_VALID_CALIBRATION_SCALE || scaleX > MAX_VALID_CALIBRATION_SCALE || scaleY > MAX_VALID_CALIBRATION_SCALE) throw new Error(`Mouse calibration produced invalid scale: (${scaleX}, ${scaleY})`);
|
|
42
|
-
return {
|
|
43
|
-
scaleX,
|
|
44
|
-
scaleY,
|
|
45
|
-
offsetX: firstActual.x - scaleX * firstInput.x,
|
|
46
|
-
offsetY: firstActual.y - scaleY * firstInput.y
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
|
-
function applyMouseCoordinateCalibration(point, calibration) {
|
|
50
|
-
return {
|
|
51
|
-
x: Math.round((point.x - calibration.offsetX) / calibration.scaleX),
|
|
52
|
-
y: Math.round((point.y - calibration.offsetY) / calibration.scaleY)
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
function mouseCoordinateCalibrationNeedsCorrection(calibration) {
|
|
56
|
-
return Math.abs(calibration.scaleX - 1) > 0.01 || Math.abs(calibration.scaleY - 1) > 0.01 || Math.abs(calibration.offsetX) > MOUSE_COORDINATE_TOLERANCE_PX || Math.abs(calibration.offsetY) > MOUSE_COORDINATE_TOLERANCE_PX;
|
|
57
|
-
}
|
|
58
|
-
function getMouseCoordinateDrift(expected, actual) {
|
|
59
|
-
return {
|
|
60
|
-
x: actual.x - expected.x,
|
|
61
|
-
y: actual.y - expected.y
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
function mouseCoordinateDriftIsWithinTolerance(drift) {
|
|
65
|
-
return Math.abs(drift.x) <= MOUSE_COORDINATE_TOLERANCE_PX && Math.abs(drift.y) <= MOUSE_COORDINATE_TOLERANCE_PX;
|
|
66
|
-
}
|
|
67
|
-
function mouseCoordinateCorrectionPoint(requested, drift) {
|
|
68
|
-
return {
|
|
69
|
-
x: requested.x - drift.x,
|
|
70
|
-
y: requested.y - drift.y
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
22
|
function _define_property(obj, key, value) {
|
|
74
23
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
75
24
|
value: value,
|
|
@@ -80,9 +29,6 @@ function _define_property(obj, key, value) {
|
|
|
80
29
|
else obj[key] = value;
|
|
81
30
|
return obj;
|
|
82
31
|
}
|
|
83
|
-
const CALIBRATION_SETTLE_DELAY_MS = 80;
|
|
84
|
-
const MOUSE_CORRECTION_SETTLE_DELAY_MS = 50;
|
|
85
|
-
const MAX_MOUSE_CORRECTION_ATTEMPTS = 3;
|
|
86
32
|
class ComputerInputDriver {
|
|
87
33
|
destroy() {
|
|
88
34
|
if (this.destroyed) return;
|
|
@@ -96,82 +42,7 @@ class ComputerInputDriver {
|
|
|
96
42
|
return this.getLibnutOrThrow('getMousePos').getMousePos();
|
|
97
43
|
}
|
|
98
44
|
moveMouse(x, y) {
|
|
99
|
-
|
|
100
|
-
x,
|
|
101
|
-
y
|
|
102
|
-
}, this.mouseCoordinateCalibration) : {
|
|
103
|
-
x,
|
|
104
|
-
y
|
|
105
|
-
};
|
|
106
|
-
this.getLibnutOrThrow('moveMouse').moveMouse(target.x, target.y);
|
|
107
|
-
}
|
|
108
|
-
async calibrateMouseCoordinates(bounds) {
|
|
109
|
-
this.assertActive('calibrateMouseCoordinates');
|
|
110
|
-
assertValidMouseCalibrationBounds(bounds);
|
|
111
|
-
const firstInput = mouseCalibrationPoint(bounds, 0.1);
|
|
112
|
-
const secondInput = mouseCalibrationPoint(bounds, 0.3);
|
|
113
|
-
const savedPosition = this.getMousePos();
|
|
114
|
-
this.mouseCoordinateCalibration = void 0;
|
|
115
|
-
try {
|
|
116
|
-
this.moveMouse(firstInput.x, firstInput.y);
|
|
117
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
118
|
-
const firstActual = this.getMousePos();
|
|
119
|
-
this.moveMouse(secondInput.x, secondInput.y);
|
|
120
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
121
|
-
const secondActual = this.getMousePos();
|
|
122
|
-
const calibration = calculateMouseCoordinateCalibration(firstInput, firstActual, secondInput, secondActual);
|
|
123
|
-
const needsCorrection = mouseCoordinateCalibrationNeedsCorrection(calibration);
|
|
124
|
-
this.mouseCoordinateCalibration = needsCorrection ? calibration : void 0;
|
|
125
|
-
let verificationDrift = {
|
|
126
|
-
x: 0,
|
|
127
|
-
y: 0
|
|
128
|
-
};
|
|
129
|
-
if (needsCorrection) {
|
|
130
|
-
const verificationTarget = mouseCalibrationPoint(bounds, 0.5);
|
|
131
|
-
this.moveMouse(verificationTarget.x, verificationTarget.y);
|
|
132
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
133
|
-
verificationDrift = this.assertMousePosition(verificationTarget.x, verificationTarget.y, 'Mouse coordinate calibration verification');
|
|
134
|
-
}
|
|
135
|
-
this.options.debug(needsCorrection ? `Mouse coordinate calibration applied: scale=(${calibration.scaleX.toFixed(4)}, ${calibration.scaleY.toFixed(4)}), offset=(${calibration.offsetX.toFixed(1)}, ${calibration.offsetY.toFixed(1)}), verification drift=(${verificationDrift.x}, ${verificationDrift.y})` : 'Mouse coordinate calibration is identity');
|
|
136
|
-
this.moveMouse(savedPosition.x, savedPosition.y);
|
|
137
|
-
} catch (error) {
|
|
138
|
-
try {
|
|
139
|
-
this.moveMouse(savedPosition.x, savedPosition.y);
|
|
140
|
-
} finally{
|
|
141
|
-
this.mouseCoordinateCalibration = void 0;
|
|
142
|
-
}
|
|
143
|
-
throw error;
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
assertMousePosition(targetX, targetY, context) {
|
|
147
|
-
const current = this.getMousePos();
|
|
148
|
-
const drift = getMouseCoordinateDrift({
|
|
149
|
-
x: targetX,
|
|
150
|
-
y: targetY
|
|
151
|
-
}, current);
|
|
152
|
-
if (!mouseCoordinateDriftIsWithinTolerance(drift)) throw new Error(`${context}: expected (${targetX}, ${targetY}), got (${current.x}, ${current.y}), drift=(${drift.x}, ${drift.y})`);
|
|
153
|
-
return drift;
|
|
154
|
-
}
|
|
155
|
-
async correctMousePosition(targetX, targetY, context) {
|
|
156
|
-
const target = {
|
|
157
|
-
x: targetX,
|
|
158
|
-
y: targetY
|
|
159
|
-
};
|
|
160
|
-
let requested = target;
|
|
161
|
-
for(let attempt = 0; attempt <= MAX_MOUSE_CORRECTION_ATTEMPTS; attempt++){
|
|
162
|
-
const current = this.getMousePos();
|
|
163
|
-
const drift = getMouseCoordinateDrift(target, current);
|
|
164
|
-
if (mouseCoordinateDriftIsWithinTolerance(drift)) {
|
|
165
|
-
if (attempt > 0) this.options.debug(`${context}: pointer correction converged after ${attempt} attempt(s), drift=(${drift.x}, ${drift.y})`);
|
|
166
|
-
return drift;
|
|
167
|
-
}
|
|
168
|
-
if (attempt === MAX_MOUSE_CORRECTION_ATTEMPTS) return this.assertMousePosition(targetX, targetY, context);
|
|
169
|
-
requested = mouseCoordinateCorrectionPoint(requested, drift);
|
|
170
|
-
this.options.debug(`${context}: correcting pointer drift=(${drift.x}, ${drift.y}) with requested point=(${requested.x}, ${requested.y}), attempt=${attempt + 1}/${MAX_MOUSE_CORRECTION_ATTEMPTS}`);
|
|
171
|
-
this.moveMouse(requested.x, requested.y);
|
|
172
|
-
await this.delay(MOUSE_CORRECTION_SETTLE_DELAY_MS);
|
|
173
|
-
}
|
|
174
|
-
throw new Error(`${context}: mouse correction ended unexpectedly`);
|
|
45
|
+
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
175
46
|
}
|
|
176
47
|
focusActiveWindow() {
|
|
177
48
|
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
@@ -307,7 +178,6 @@ class ComputerInputDriver {
|
|
|
307
178
|
constructor(options){
|
|
308
179
|
_define_property(this, "options", void 0);
|
|
309
180
|
_define_property(this, "destroyed", void 0);
|
|
310
|
-
_define_property(this, "mouseCoordinateCalibration", void 0);
|
|
311
181
|
_define_property(this, "pendingInputDelayWaits", void 0);
|
|
312
182
|
this.options = options;
|
|
313
183
|
this.destroyed = false;
|
|
@@ -315,7 +185,50 @@ class ComputerInputDriver {
|
|
|
315
185
|
}
|
|
316
186
|
}
|
|
317
187
|
const debugXvfb = getDebug('computer:xvfb');
|
|
318
|
-
|
|
188
|
+
const xvfbCleanupMonitorScript = String.raw`
|
|
189
|
+
const parentPid = Number(process.argv[1]);
|
|
190
|
+
const xvfbPid = Number(process.argv[2]);
|
|
191
|
+
const timer = setInterval(() => {
|
|
192
|
+
try {
|
|
193
|
+
process.kill(xvfbPid, 0);
|
|
194
|
+
} catch {
|
|
195
|
+
clearInterval(timer);
|
|
196
|
+
process.exit(0);
|
|
197
|
+
}
|
|
198
|
+
try {
|
|
199
|
+
process.kill(parentPid, 0);
|
|
200
|
+
return;
|
|
201
|
+
} catch {
|
|
202
|
+
// The owner is gone, so its X11 clients can no longer receive XIO errors.
|
|
203
|
+
}
|
|
204
|
+
try {
|
|
205
|
+
process.kill(xvfbPid, 'SIGTERM');
|
|
206
|
+
} catch {
|
|
207
|
+
// Xvfb may have already exited.
|
|
208
|
+
}
|
|
209
|
+
clearInterval(timer);
|
|
210
|
+
}, 100);
|
|
211
|
+
`;
|
|
212
|
+
function scheduleXvfbStopAfterProcessExit(instance, parentPid = process.pid) {
|
|
213
|
+
const xvfbPid = instance.process.pid;
|
|
214
|
+
if (!xvfbPid) throw new Error('Cannot schedule Xvfb cleanup before its process starts');
|
|
215
|
+
const monitor = spawn(process.execPath, [
|
|
216
|
+
'-e',
|
|
217
|
+
xvfbCleanupMonitorScript,
|
|
218
|
+
String(parentPid),
|
|
219
|
+
String(xvfbPid)
|
|
220
|
+
], {
|
|
221
|
+
detached: true,
|
|
222
|
+
stdio: 'ignore'
|
|
223
|
+
});
|
|
224
|
+
monitor.on('error', (error)=>{
|
|
225
|
+
debugXvfb(`Xvfb cleanup monitor failed: ${error.message}`);
|
|
226
|
+
});
|
|
227
|
+
instance.process.unref();
|
|
228
|
+
monitor.unref();
|
|
229
|
+
return monitor;
|
|
230
|
+
}
|
|
231
|
+
function createXvfbSignalCleanup(cleanup, source = process) {
|
|
319
232
|
return ()=>{
|
|
320
233
|
if (!hasActiveCliInterruptWaiter(source)) cleanup();
|
|
321
234
|
};
|
|
@@ -548,33 +461,19 @@ function runPowershell(script) {
|
|
|
548
461
|
windowsHide: true
|
|
549
462
|
});
|
|
550
463
|
}
|
|
551
|
-
function
|
|
464
|
+
function listWindowsDisplays() {
|
|
552
465
|
const script = `
|
|
553
466
|
Add-Type -AssemblyName System.Windows.Forms
|
|
554
467
|
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
555
|
-
$
|
|
556
|
-
[PSCustomObject]@{
|
|
557
|
-
id = $_.DeviceName
|
|
558
|
-
name = $_.DeviceName
|
|
559
|
-
primary = $_.Primary
|
|
560
|
-
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
561
|
-
}
|
|
468
|
+
[PSCustomObject]@{ id = $_.DeviceName; name = $_.DeviceName; primary = $_.Primary }
|
|
562
469
|
}
|
|
563
470
|
ConvertTo-Json @($s) -Compress
|
|
564
471
|
`.trim();
|
|
565
|
-
const
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
571
|
-
return displays;
|
|
572
|
-
}
|
|
573
|
-
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
574
|
-
return geometries.map((display)=>({
|
|
575
|
-
id: display.id,
|
|
576
|
-
name: display.name,
|
|
577
|
-
primary: display.primary
|
|
472
|
+
const parsed = JSON.parse(runPowershell(script).trim());
|
|
473
|
+
return parsed.map((d)=>({
|
|
474
|
+
id: String(d.id),
|
|
475
|
+
name: d.name || String(d.id),
|
|
476
|
+
primary: d.primary || false
|
|
578
477
|
}));
|
|
579
478
|
}
|
|
580
479
|
let device_libnut = null;
|
|
@@ -676,16 +575,6 @@ function getDisplayInfoBinary() {
|
|
|
676
575
|
function isFiniteNumber(value) {
|
|
677
576
|
return 'number' == typeof value && Number.isFinite(value);
|
|
678
577
|
}
|
|
679
|
-
function isDisplayBounds(value) {
|
|
680
|
-
if (!value || 'object' != typeof value) return false;
|
|
681
|
-
const bounds = value;
|
|
682
|
-
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
683
|
-
}
|
|
684
|
-
function isWindowsDisplayGeometry(value) {
|
|
685
|
-
if (!value || 'object' != typeof value) return false;
|
|
686
|
-
const candidate = value;
|
|
687
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
688
|
-
}
|
|
689
578
|
function isDarwinDisplayGeometry(value) {
|
|
690
579
|
if (!value || 'object' != typeof value) return false;
|
|
691
580
|
const candidate = value;
|
|
@@ -729,11 +618,8 @@ function readDarwinFrontmostApplication() {
|
|
|
729
618
|
}
|
|
730
619
|
}
|
|
731
620
|
async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
|
|
621
|
+
await inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
732
622
|
const current = inputDriver.getMousePos();
|
|
733
|
-
const drift = {
|
|
734
|
-
x: current.x - targetX,
|
|
735
|
-
y: current.y - targetY
|
|
736
|
-
};
|
|
737
623
|
debugComputerInput('tap mouse moved %o', {
|
|
738
624
|
reason,
|
|
739
625
|
target: {
|
|
@@ -741,7 +627,10 @@ async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDurati
|
|
|
741
627
|
y: targetY
|
|
742
628
|
},
|
|
743
629
|
current,
|
|
744
|
-
drift
|
|
630
|
+
drift: {
|
|
631
|
+
x: current.x - targetX,
|
|
632
|
+
y: current.y - targetY
|
|
633
|
+
}
|
|
745
634
|
});
|
|
746
635
|
await inputDriver.withMouseButton('left', async ()=>{
|
|
747
636
|
debugComputerInput('tap mouse down %o', {
|
|
@@ -760,18 +649,9 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
760
649
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
761
650
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
762
651
|
}
|
|
763
|
-
function
|
|
764
|
-
if (
|
|
765
|
-
|
|
766
|
-
return displays.find((display)=>display.id === displayId);
|
|
767
|
-
}
|
|
768
|
-
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
769
|
-
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
770
|
-
if ('win32' === process.platform) {
|
|
771
|
-
const geometry = resolveWindowsDisplayGeometryFromList(displayId, windowsDisplays ?? readWindowsDisplayGeometries());
|
|
772
|
-
if (!geometry) throw new Error(displayId ? `Requested Windows display not found: ${displayId}` : 'No Windows displays were detected');
|
|
773
|
-
return geometry;
|
|
774
|
-
}
|
|
652
|
+
function resolveDisplayGeometry(displayId) {
|
|
653
|
+
if ('darwin' !== process.platform) return;
|
|
654
|
+
return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
775
655
|
}
|
|
776
656
|
function mapDisplayLocalPointToGlobal(point, geometry) {
|
|
777
657
|
if (!geometry) return point;
|
|
@@ -857,20 +737,6 @@ function normalizePrimaryKey(key) {
|
|
|
857
737
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
858
738
|
}
|
|
859
739
|
class ComputerDevice {
|
|
860
|
-
async moveGlobalPointer(point, context, smooth) {
|
|
861
|
-
const target = {
|
|
862
|
-
x: Math.round(point.x),
|
|
863
|
-
y: Math.round(point.y)
|
|
864
|
-
};
|
|
865
|
-
if (smooth) await this.inputDriver.smoothMoveMouse(target.x, target.y, smooth.smoothSteps, smooth.smoothDelay);
|
|
866
|
-
else this.inputDriver.moveMouse(target.x, target.y);
|
|
867
|
-
await this.inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
868
|
-
if ('win32' === process.platform) await this.inputDriver.correctMousePosition(target.x, target.y, context);
|
|
869
|
-
return target;
|
|
870
|
-
}
|
|
871
|
-
moveDisplayPointer(point, context, smooth) {
|
|
872
|
-
return this.moveGlobalPointer(this.toGlobalPoint(point), context, smooth);
|
|
873
|
-
}
|
|
874
740
|
async focusKeyboardTarget(element, delayMs) {
|
|
875
741
|
const [x, y] = element.center;
|
|
876
742
|
if ('darwin' === process.platform) await this.inputPrimitives.pointer.tap({
|
|
@@ -878,10 +744,11 @@ class ComputerDevice {
|
|
|
878
744
|
y
|
|
879
745
|
});
|
|
880
746
|
else {
|
|
881
|
-
|
|
747
|
+
const point = this.toGlobalPoint({
|
|
882
748
|
x,
|
|
883
749
|
y
|
|
884
|
-
}
|
|
750
|
+
});
|
|
751
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
885
752
|
this.inputDriver.mouseClick('left');
|
|
886
753
|
}
|
|
887
754
|
await this.inputDriver.delay(delayMs);
|
|
@@ -900,7 +767,7 @@ class ComputerDevice {
|
|
|
900
767
|
}));
|
|
901
768
|
} catch (error) {
|
|
902
769
|
debugDevice(`Failed to list displays: ${error}`);
|
|
903
|
-
|
|
770
|
+
return [];
|
|
904
771
|
}
|
|
905
772
|
}
|
|
906
773
|
async connect() {
|
|
@@ -912,24 +779,26 @@ class ComputerDevice {
|
|
|
912
779
|
this.xvfbInstance = await startXvfb({
|
|
913
780
|
resolution: this.options?.xvfbResolution
|
|
914
781
|
});
|
|
782
|
+
if (this.options?.keepXvfbAliveUntilProcessExit) scheduleXvfbStopAfterProcessExit(this.xvfbInstance);
|
|
915
783
|
process.env.DISPLAY = this.xvfbInstance.display;
|
|
916
784
|
debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
|
|
917
|
-
this.
|
|
918
|
-
|
|
919
|
-
this.xvfbInstance
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
785
|
+
if (!this.options?.keepXvfbAliveUntilProcessExit) {
|
|
786
|
+
this.xvfbCleanup = ()=>{
|
|
787
|
+
if (this.xvfbInstance) {
|
|
788
|
+
this.xvfbInstance.stop();
|
|
789
|
+
this.xvfbInstance = void 0;
|
|
790
|
+
}
|
|
791
|
+
};
|
|
792
|
+
this.xvfbSignalCleanup = createXvfbSignalCleanup(()=>this.xvfbCleanup?.());
|
|
793
|
+
process.on('exit', this.xvfbCleanup);
|
|
794
|
+
process.on('SIGINT', this.xvfbSignalCleanup);
|
|
795
|
+
process.on('SIGTERM', this.xvfbSignalCleanup);
|
|
796
|
+
}
|
|
927
797
|
}
|
|
928
798
|
device_libnut = await getLibnut();
|
|
929
|
-
|
|
930
|
-
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
799
|
+
this.displayGeometry = resolveDisplayGeometry(this.displayId);
|
|
931
800
|
const size = await this.size();
|
|
932
|
-
const displays =
|
|
801
|
+
const displays = await ComputerDevice.listDisplays();
|
|
933
802
|
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
934
803
|
this.description = `
|
|
935
804
|
Type: Computer
|
|
@@ -939,28 +808,28 @@ Screen Size: ${size.width}x${size.height}
|
|
|
939
808
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
940
809
|
`;
|
|
941
810
|
debugDevice('Computer device connected', this.description);
|
|
942
|
-
await this.healthCheck(
|
|
811
|
+
await this.healthCheck();
|
|
943
812
|
} catch (error) {
|
|
944
813
|
if (this.xvfbInstance) {
|
|
945
|
-
this.xvfbInstance.stop();
|
|
814
|
+
if (!this.options?.keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
946
815
|
this.xvfbInstance = void 0;
|
|
947
816
|
}
|
|
948
817
|
if (this.xvfbCleanup) {
|
|
949
818
|
process.removeListener('exit', this.xvfbCleanup);
|
|
950
|
-
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
951
819
|
this.xvfbCleanup = void 0;
|
|
952
820
|
}
|
|
953
|
-
if (this.
|
|
954
|
-
process.removeListener('SIGINT', this.
|
|
955
|
-
this.
|
|
821
|
+
if (this.xvfbSignalCleanup) {
|
|
822
|
+
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
823
|
+
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
824
|
+
this.xvfbSignalCleanup = void 0;
|
|
956
825
|
}
|
|
957
826
|
debugDevice(`Failed to connect: ${error}`);
|
|
958
827
|
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
959
828
|
}
|
|
960
829
|
}
|
|
961
|
-
async healthCheck(
|
|
830
|
+
async healthCheck() {
|
|
962
831
|
console.log('[HealthCheck] Starting health check...');
|
|
963
|
-
console.log("[HealthCheck] @midscene/computer v1.12.2
|
|
832
|
+
console.log("[HealthCheck] @midscene/computer v1.12.2");
|
|
964
833
|
console.log('[HealthCheck] Taking screenshot...');
|
|
965
834
|
const screenshotTimeout = 15000;
|
|
966
835
|
let timeoutId;
|
|
@@ -972,27 +841,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
972
841
|
timeoutPromise
|
|
973
842
|
]);
|
|
974
843
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
975
|
-
console.log('[HealthCheck]
|
|
844
|
+
console.log('[HealthCheck] Moving mouse...');
|
|
976
845
|
const startPos = this.inputDriver.getMousePos();
|
|
977
846
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
} finally{
|
|
993
|
-
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
994
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
995
|
-
}
|
|
847
|
+
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
848
|
+
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
849
|
+
const targetX = startPos.x + offsetX;
|
|
850
|
+
const targetY = startPos.y + offsetY;
|
|
851
|
+
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
852
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
853
|
+
await sleep(50);
|
|
854
|
+
const movedPos = this.inputDriver.getMousePos();
|
|
855
|
+
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
856
|
+
const deltaX = Math.abs(movedPos.x - targetX);
|
|
857
|
+
const deltaY = Math.abs(movedPos.y - targetY);
|
|
858
|
+
if (deltaX > 5 || deltaY > 5) {
|
|
859
|
+
const msg = `[HealthCheck] WARNING: Mouse control may not be working. Expected (${targetX}, ${targetY}), got (${movedPos.x}, ${movedPos.y}), delta=(${deltaX}, ${deltaY})`;
|
|
860
|
+
warnDevice(msg);
|
|
996
861
|
}
|
|
997
862
|
if ('win32' === process.platform && !this.isRunningAsAdmin()) {
|
|
998
863
|
const hint = [
|
|
@@ -1002,7 +867,10 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1002
867
|
].join(' ');
|
|
1003
868
|
warnDevice(`[HealthCheck] ${hint}`);
|
|
1004
869
|
}
|
|
870
|
+
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
871
|
+
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
1005
872
|
console.log('[HealthCheck] Listing monitors...');
|
|
873
|
+
const displays = await ComputerDevice.listDisplays();
|
|
1006
874
|
if (displays.length > 0) {
|
|
1007
875
|
console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
|
|
1008
876
|
for (const display of displays){
|
|
@@ -1185,14 +1053,10 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1185
1053
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1186
1054
|
if ('win32' === process.platform) {
|
|
1187
1055
|
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
1188
|
-
if (activeWindowRect) {
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
};
|
|
1193
|
-
const bounds = this.displayGeometry?.bounds;
|
|
1194
|
-
if (!bounds || activeWindowCenter.x >= bounds.x && activeWindowCenter.x < bounds.x + bounds.width && activeWindowCenter.y >= bounds.y && activeWindowCenter.y < bounds.y + bounds.height) return activeWindowCenter;
|
|
1195
|
-
}
|
|
1056
|
+
if (activeWindowRect) return {
|
|
1057
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
1058
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
1059
|
+
};
|
|
1196
1060
|
}
|
|
1197
1061
|
return this.toGlobalPoint({
|
|
1198
1062
|
x: screenSize.width / 2,
|
|
@@ -1203,16 +1067,17 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1203
1067
|
if (param.locate) {
|
|
1204
1068
|
const element = param.locate;
|
|
1205
1069
|
const [x, y] = element.center;
|
|
1206
|
-
|
|
1070
|
+
const point = this.toGlobalPoint({
|
|
1207
1071
|
x,
|
|
1208
1072
|
y
|
|
1209
|
-
}
|
|
1073
|
+
});
|
|
1074
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1210
1075
|
return;
|
|
1211
1076
|
}
|
|
1212
1077
|
const screenSize = await this.size();
|
|
1213
1078
|
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
1214
1079
|
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
1215
|
-
|
|
1080
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1216
1081
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1217
1082
|
return screenSize;
|
|
1218
1083
|
}
|
|
@@ -1301,18 +1166,19 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1301
1166
|
if (this.destroyed) return;
|
|
1302
1167
|
this.destroyed = true;
|
|
1303
1168
|
this.inputDriver.destroy();
|
|
1169
|
+
const keepXvfbAliveUntilProcessExit = this.options?.keepXvfbAliveUntilProcessExit === true;
|
|
1304
1170
|
if (this.xvfbInstance) {
|
|
1305
|
-
this.xvfbInstance.stop();
|
|
1171
|
+
if (!keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
1306
1172
|
this.xvfbInstance = void 0;
|
|
1307
1173
|
}
|
|
1308
|
-
if (this.xvfbCleanup) {
|
|
1174
|
+
if (this.xvfbCleanup && !keepXvfbAliveUntilProcessExit) {
|
|
1309
1175
|
process.removeListener('exit', this.xvfbCleanup);
|
|
1310
|
-
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
1311
1176
|
this.xvfbCleanup = void 0;
|
|
1312
1177
|
}
|
|
1313
|
-
if (this.
|
|
1314
|
-
process.removeListener('SIGINT', this.
|
|
1315
|
-
this.
|
|
1178
|
+
if (this.xvfbSignalCleanup) {
|
|
1179
|
+
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
1180
|
+
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
1181
|
+
this.xvfbSignalCleanup = void 0;
|
|
1316
1182
|
}
|
|
1317
1183
|
debugDevice('Computer device destroyed');
|
|
1318
1184
|
}
|
|
@@ -1328,7 +1194,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1328
1194
|
device_define_property(this, "destroyed", false);
|
|
1329
1195
|
device_define_property(this, "xvfbInstance", void 0);
|
|
1330
1196
|
device_define_property(this, "xvfbCleanup", void 0);
|
|
1331
|
-
device_define_property(this, "
|
|
1197
|
+
device_define_property(this, "xvfbSignalCleanup", void 0);
|
|
1332
1198
|
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
1333
1199
|
getLibnut: ()=>device_libnut,
|
|
1334
1200
|
useAppleScript: ()=>this.useAppleScript,
|
|
@@ -1360,16 +1226,14 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1360
1226
|
},
|
|
1361
1227
|
holdDuration,
|
|
1362
1228
|
displayId: this.displayId,
|
|
1363
|
-
displayGeometry: this.displayGeometry
|
|
1229
|
+
displayGeometry: this.displayGeometry ? {
|
|
1230
|
+
screenIndex: this.displayGeometry.screenIndex,
|
|
1231
|
+
cgDisplayId: this.displayGeometry.cgDisplayId,
|
|
1232
|
+
bounds: this.displayGeometry.bounds
|
|
1233
|
+
} : void 0
|
|
1364
1234
|
});
|
|
1365
1235
|
const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
|
|
1366
|
-
await this.
|
|
1367
|
-
x: targetX,
|
|
1368
|
-
y: targetY
|
|
1369
|
-
}, 'Mouse did not reach the tap target', {
|
|
1370
|
-
smoothSteps: SMOOTH_MOVE_STEPS_TAP,
|
|
1371
|
-
smoothDelay: SMOOTH_MOVE_DELAY_TAP
|
|
1372
|
-
});
|
|
1236
|
+
await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
1373
1237
|
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'primary');
|
|
1374
1238
|
if (frontmostBefore && 'darwin' === process.platform) {
|
|
1375
1239
|
await sleep(CLICK_FOCUS_SETTLE_DELAY);
|
|
@@ -1381,43 +1245,42 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1381
1245
|
focusChanged
|
|
1382
1246
|
});
|
|
1383
1247
|
if (focusChanged) {
|
|
1384
|
-
|
|
1385
|
-
x: targetX,
|
|
1386
|
-
y: targetY
|
|
1387
|
-
}, 'Mouse did not reach the focus follow-up target');
|
|
1248
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
1388
1249
|
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'focus-follow-up');
|
|
1389
1250
|
}
|
|
1390
1251
|
}
|
|
1391
1252
|
},
|
|
1392
1253
|
doubleClick: async ({ x, y })=>{
|
|
1393
|
-
|
|
1254
|
+
const target = this.toGlobalPoint({
|
|
1394
1255
|
x,
|
|
1395
1256
|
y
|
|
1396
|
-
}
|
|
1257
|
+
});
|
|
1258
|
+
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1397
1259
|
this.inputDriver.mouseClick('left', true);
|
|
1398
1260
|
},
|
|
1399
1261
|
rightClick: async ({ x, y })=>{
|
|
1400
|
-
|
|
1262
|
+
const target = this.toGlobalPoint({
|
|
1401
1263
|
x,
|
|
1402
1264
|
y
|
|
1403
|
-
}
|
|
1265
|
+
});
|
|
1266
|
+
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1404
1267
|
this.inputDriver.mouseClick('right');
|
|
1405
1268
|
},
|
|
1406
1269
|
hover: async ({ x, y })=>{
|
|
1407
|
-
|
|
1270
|
+
const target = this.toGlobalPoint({
|
|
1408
1271
|
x,
|
|
1409
1272
|
y
|
|
1410
|
-
}, 'Mouse did not reach the hover target', {
|
|
1411
|
-
smoothSteps: SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1412
|
-
smoothDelay: SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
1413
1273
|
});
|
|
1274
|
+
await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
1414
1275
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1415
1276
|
},
|
|
1416
1277
|
dragAndDrop: async (from, to)=>{
|
|
1417
|
-
|
|
1278
|
+
const globalFrom = this.toGlobalPoint(from);
|
|
1279
|
+
const globalTo = this.toGlobalPoint(to);
|
|
1280
|
+
this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
|
|
1418
1281
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1419
1282
|
await this.inputDriver.delay(100);
|
|
1420
|
-
|
|
1283
|
+
this.inputDriver.moveMouse(Math.round(globalTo.x), Math.round(globalTo.y));
|
|
1421
1284
|
await this.inputDriver.delay(100);
|
|
1422
1285
|
});
|
|
1423
1286
|
}
|
|
@@ -2147,7 +2010,8 @@ function createLocalComputerDevice(opts) {
|
|
|
2147
2010
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2148
2011
|
keyboardDriver: opts?.keyboardDriver,
|
|
2149
2012
|
headless: opts?.headless,
|
|
2150
|
-
xvfbResolution: opts?.xvfbResolution
|
|
2013
|
+
xvfbResolution: opts?.xvfbResolution,
|
|
2014
|
+
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
2151
2015
|
});
|
|
2152
2016
|
}
|
|
2153
2017
|
function createRDPComputerDevice(opts) {
|
|
@@ -2290,6 +2154,9 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2290
2154
|
...void 0 !== keyboardTypeDelay ? {
|
|
2291
2155
|
keyboardTypeDelay
|
|
2292
2156
|
} : {},
|
|
2157
|
+
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2158
|
+
keepXvfbAliveUntilProcessExit: true
|
|
2159
|
+
} : {},
|
|
2293
2160
|
...extractAgentBehaviorInitArgs(opts) ?? {},
|
|
2294
2161
|
...reportOptions ?? {}
|
|
2295
2162
|
};
|
|
@@ -2355,21 +2222,23 @@ class ComputerMidsceneTools extends BaseMidsceneTools {
|
|
|
2355
2222
|
}
|
|
2356
2223
|
];
|
|
2357
2224
|
}
|
|
2358
|
-
constructor(
|
|
2359
|
-
super(
|
|
2225
|
+
constructor(options = {}){
|
|
2226
|
+
super(), agent_tools_define_property(this, "options", void 0), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", void 0), this.options = options, this.initArgSpec = {
|
|
2360
2227
|
namespace: 'computer',
|
|
2361
2228
|
shape: computerInitArgShape,
|
|
2362
2229
|
cli: {
|
|
2363
2230
|
preferBareKeys: true
|
|
2364
2231
|
},
|
|
2365
2232
|
adapt: (extracted)=>adaptComputerInitArgs(extracted)
|
|
2366
|
-
}
|
|
2233
|
+
};
|
|
2367
2234
|
}
|
|
2368
2235
|
}
|
|
2369
|
-
const tools = new ComputerMidsceneTools(
|
|
2236
|
+
const tools = new ComputerMidsceneTools({
|
|
2237
|
+
keepXvfbAliveUntilProcessExit: true
|
|
2238
|
+
});
|
|
2370
2239
|
runToolsCLI(tools, 'midscene-computer', {
|
|
2371
2240
|
stripPrefix: 'computer_',
|
|
2372
|
-
version: "1.12.2
|
|
2241
|
+
version: "1.12.2",
|
|
2373
2242
|
extraCommands: createReportCliCommands()
|
|
2374
2243
|
}).catch((e)=>{
|
|
2375
2244
|
process.exit(reportCLIError(e));
|