@midscene/computer 1.12.2-beta-20260828072235.0 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +152 -283
- package/dist/es/index.mjs +149 -282
- package/dist/lib/cli.js +152 -283
- package/dist/lib/index.js +149 -282
- package/dist/types/index.d.ts +20 -6
- package/package.json +3 -3
package/dist/lib/cli.js
CHANGED
|
@@ -44,57 +44,6 @@ const external_screenshot_desktop_namespaceObject = require("screenshot-desktop"
|
|
|
44
44
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
45
45
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
46
46
|
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
47
|
-
const MOUSE_COORDINATE_TOLERANCE_PX = 5;
|
|
48
|
-
const MIN_CALIBRATION_AXIS_DELTA = 20;
|
|
49
|
-
const MIN_VALID_CALIBRATION_SCALE = 0.1;
|
|
50
|
-
const MAX_VALID_CALIBRATION_SCALE = 10;
|
|
51
|
-
function assertValidMouseCalibrationBounds(bounds) {
|
|
52
|
-
if (!Number.isFinite(bounds.x) || !Number.isFinite(bounds.y) || !Number.isFinite(bounds.width) || !Number.isFinite(bounds.height) || bounds.width <= 0 || bounds.height <= 0) throw new Error(`Mouse calibration bounds are invalid: (${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`);
|
|
53
|
-
}
|
|
54
|
-
function mouseCalibrationPoint(bounds, ratio) {
|
|
55
|
-
return {
|
|
56
|
-
x: Math.round(bounds.x + bounds.width * ratio),
|
|
57
|
-
y: Math.round(bounds.y + bounds.height * ratio)
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
function calculateMouseCoordinateCalibration(firstInput, firstActual, secondInput, secondActual) {
|
|
61
|
-
const inputDeltaX = secondInput.x - firstInput.x;
|
|
62
|
-
const inputDeltaY = secondInput.y - firstInput.y;
|
|
63
|
-
if (Math.abs(inputDeltaX) < MIN_CALIBRATION_AXIS_DELTA || Math.abs(inputDeltaY) < MIN_CALIBRATION_AXIS_DELTA) throw new Error(`Mouse calibration points are too close: delta=(${inputDeltaX}, ${inputDeltaY})`);
|
|
64
|
-
const scaleX = (secondActual.x - firstActual.x) / inputDeltaX;
|
|
65
|
-
const scaleY = (secondActual.y - firstActual.y) / inputDeltaY;
|
|
66
|
-
if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY) || scaleX < MIN_VALID_CALIBRATION_SCALE || scaleY < MIN_VALID_CALIBRATION_SCALE || scaleX > MAX_VALID_CALIBRATION_SCALE || scaleY > MAX_VALID_CALIBRATION_SCALE) throw new Error(`Mouse calibration produced invalid scale: (${scaleX}, ${scaleY})`);
|
|
67
|
-
return {
|
|
68
|
-
scaleX,
|
|
69
|
-
scaleY,
|
|
70
|
-
offsetX: firstActual.x - scaleX * firstInput.x,
|
|
71
|
-
offsetY: firstActual.y - scaleY * firstInput.y
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
function applyMouseCoordinateCalibration(point, calibration) {
|
|
75
|
-
return {
|
|
76
|
-
x: Math.round((point.x - calibration.offsetX) / calibration.scaleX),
|
|
77
|
-
y: Math.round((point.y - calibration.offsetY) / calibration.scaleY)
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
function mouseCoordinateCalibrationNeedsCorrection(calibration) {
|
|
81
|
-
return Math.abs(calibration.scaleX - 1) > 0.01 || Math.abs(calibration.scaleY - 1) > 0.01 || Math.abs(calibration.offsetX) > MOUSE_COORDINATE_TOLERANCE_PX || Math.abs(calibration.offsetY) > MOUSE_COORDINATE_TOLERANCE_PX;
|
|
82
|
-
}
|
|
83
|
-
function getMouseCoordinateDrift(expected, actual) {
|
|
84
|
-
return {
|
|
85
|
-
x: actual.x - expected.x,
|
|
86
|
-
y: actual.y - expected.y
|
|
87
|
-
};
|
|
88
|
-
}
|
|
89
|
-
function mouseCoordinateDriftIsWithinTolerance(drift) {
|
|
90
|
-
return Math.abs(drift.x) <= MOUSE_COORDINATE_TOLERANCE_PX && Math.abs(drift.y) <= MOUSE_COORDINATE_TOLERANCE_PX;
|
|
91
|
-
}
|
|
92
|
-
function mouseCoordinateCorrectionPoint(requested, drift) {
|
|
93
|
-
return {
|
|
94
|
-
x: requested.x - drift.x,
|
|
95
|
-
y: requested.y - drift.y
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
47
|
function _define_property(obj, key, value) {
|
|
99
48
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
100
49
|
value: value,
|
|
@@ -105,9 +54,6 @@ function _define_property(obj, key, value) {
|
|
|
105
54
|
else obj[key] = value;
|
|
106
55
|
return obj;
|
|
107
56
|
}
|
|
108
|
-
const CALIBRATION_SETTLE_DELAY_MS = 80;
|
|
109
|
-
const MOUSE_CORRECTION_SETTLE_DELAY_MS = 50;
|
|
110
|
-
const MAX_MOUSE_CORRECTION_ATTEMPTS = 3;
|
|
111
57
|
class ComputerInputDriver {
|
|
112
58
|
destroy() {
|
|
113
59
|
if (this.destroyed) return;
|
|
@@ -121,82 +67,7 @@ class ComputerInputDriver {
|
|
|
121
67
|
return this.getLibnutOrThrow('getMousePos').getMousePos();
|
|
122
68
|
}
|
|
123
69
|
moveMouse(x, y) {
|
|
124
|
-
|
|
125
|
-
x,
|
|
126
|
-
y
|
|
127
|
-
}, this.mouseCoordinateCalibration) : {
|
|
128
|
-
x,
|
|
129
|
-
y
|
|
130
|
-
};
|
|
131
|
-
this.getLibnutOrThrow('moveMouse').moveMouse(target.x, target.y);
|
|
132
|
-
}
|
|
133
|
-
async calibrateMouseCoordinates(bounds) {
|
|
134
|
-
this.assertActive('calibrateMouseCoordinates');
|
|
135
|
-
assertValidMouseCalibrationBounds(bounds);
|
|
136
|
-
const firstInput = mouseCalibrationPoint(bounds, 0.1);
|
|
137
|
-
const secondInput = mouseCalibrationPoint(bounds, 0.3);
|
|
138
|
-
const savedPosition = this.getMousePos();
|
|
139
|
-
this.mouseCoordinateCalibration = void 0;
|
|
140
|
-
try {
|
|
141
|
-
this.moveMouse(firstInput.x, firstInput.y);
|
|
142
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
143
|
-
const firstActual = this.getMousePos();
|
|
144
|
-
this.moveMouse(secondInput.x, secondInput.y);
|
|
145
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
146
|
-
const secondActual = this.getMousePos();
|
|
147
|
-
const calibration = calculateMouseCoordinateCalibration(firstInput, firstActual, secondInput, secondActual);
|
|
148
|
-
const needsCorrection = mouseCoordinateCalibrationNeedsCorrection(calibration);
|
|
149
|
-
this.mouseCoordinateCalibration = needsCorrection ? calibration : void 0;
|
|
150
|
-
let verificationDrift = {
|
|
151
|
-
x: 0,
|
|
152
|
-
y: 0
|
|
153
|
-
};
|
|
154
|
-
if (needsCorrection) {
|
|
155
|
-
const verificationTarget = mouseCalibrationPoint(bounds, 0.5);
|
|
156
|
-
this.moveMouse(verificationTarget.x, verificationTarget.y);
|
|
157
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
158
|
-
verificationDrift = this.assertMousePosition(verificationTarget.x, verificationTarget.y, 'Mouse coordinate calibration verification');
|
|
159
|
-
}
|
|
160
|
-
this.options.debug(needsCorrection ? `Mouse coordinate calibration applied: scale=(${calibration.scaleX.toFixed(4)}, ${calibration.scaleY.toFixed(4)}), offset=(${calibration.offsetX.toFixed(1)}, ${calibration.offsetY.toFixed(1)}), verification drift=(${verificationDrift.x}, ${verificationDrift.y})` : 'Mouse coordinate calibration is identity');
|
|
161
|
-
this.moveMouse(savedPosition.x, savedPosition.y);
|
|
162
|
-
} catch (error) {
|
|
163
|
-
try {
|
|
164
|
-
this.moveMouse(savedPosition.x, savedPosition.y);
|
|
165
|
-
} finally{
|
|
166
|
-
this.mouseCoordinateCalibration = void 0;
|
|
167
|
-
}
|
|
168
|
-
throw error;
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
assertMousePosition(targetX, targetY, context) {
|
|
172
|
-
const current = this.getMousePos();
|
|
173
|
-
const drift = getMouseCoordinateDrift({
|
|
174
|
-
x: targetX,
|
|
175
|
-
y: targetY
|
|
176
|
-
}, current);
|
|
177
|
-
if (!mouseCoordinateDriftIsWithinTolerance(drift)) throw new Error(`${context}: expected (${targetX}, ${targetY}), got (${current.x}, ${current.y}), drift=(${drift.x}, ${drift.y})`);
|
|
178
|
-
return drift;
|
|
179
|
-
}
|
|
180
|
-
async correctMousePosition(targetX, targetY, context) {
|
|
181
|
-
const target = {
|
|
182
|
-
x: targetX,
|
|
183
|
-
y: targetY
|
|
184
|
-
};
|
|
185
|
-
let requested = target;
|
|
186
|
-
for(let attempt = 0; attempt <= MAX_MOUSE_CORRECTION_ATTEMPTS; attempt++){
|
|
187
|
-
const current = this.getMousePos();
|
|
188
|
-
const drift = getMouseCoordinateDrift(target, current);
|
|
189
|
-
if (mouseCoordinateDriftIsWithinTolerance(drift)) {
|
|
190
|
-
if (attempt > 0) this.options.debug(`${context}: pointer correction converged after ${attempt} attempt(s), drift=(${drift.x}, ${drift.y})`);
|
|
191
|
-
return drift;
|
|
192
|
-
}
|
|
193
|
-
if (attempt === MAX_MOUSE_CORRECTION_ATTEMPTS) return this.assertMousePosition(targetX, targetY, context);
|
|
194
|
-
requested = mouseCoordinateCorrectionPoint(requested, drift);
|
|
195
|
-
this.options.debug(`${context}: correcting pointer drift=(${drift.x}, ${drift.y}) with requested point=(${requested.x}, ${requested.y}), attempt=${attempt + 1}/${MAX_MOUSE_CORRECTION_ATTEMPTS}`);
|
|
196
|
-
this.moveMouse(requested.x, requested.y);
|
|
197
|
-
await this.delay(MOUSE_CORRECTION_SETTLE_DELAY_MS);
|
|
198
|
-
}
|
|
199
|
-
throw new Error(`${context}: mouse correction ended unexpectedly`);
|
|
70
|
+
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
200
71
|
}
|
|
201
72
|
focusActiveWindow() {
|
|
202
73
|
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
@@ -332,7 +203,6 @@ class ComputerInputDriver {
|
|
|
332
203
|
constructor(options){
|
|
333
204
|
_define_property(this, "options", void 0);
|
|
334
205
|
_define_property(this, "destroyed", void 0);
|
|
335
|
-
_define_property(this, "mouseCoordinateCalibration", void 0);
|
|
336
206
|
_define_property(this, "pendingInputDelayWaits", void 0);
|
|
337
207
|
this.options = options;
|
|
338
208
|
this.destroyed = false;
|
|
@@ -341,7 +211,50 @@ class ComputerInputDriver {
|
|
|
341
211
|
}
|
|
342
212
|
const interrupt_namespaceObject = require("@midscene/shared/cli/interrupt");
|
|
343
213
|
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
344
|
-
|
|
214
|
+
const xvfbCleanupMonitorScript = String.raw`
|
|
215
|
+
const parentPid = Number(process.argv[1]);
|
|
216
|
+
const xvfbPid = Number(process.argv[2]);
|
|
217
|
+
const timer = setInterval(() => {
|
|
218
|
+
try {
|
|
219
|
+
process.kill(xvfbPid, 0);
|
|
220
|
+
} catch {
|
|
221
|
+
clearInterval(timer);
|
|
222
|
+
process.exit(0);
|
|
223
|
+
}
|
|
224
|
+
try {
|
|
225
|
+
process.kill(parentPid, 0);
|
|
226
|
+
return;
|
|
227
|
+
} catch {
|
|
228
|
+
// The owner is gone, so its X11 clients can no longer receive XIO errors.
|
|
229
|
+
}
|
|
230
|
+
try {
|
|
231
|
+
process.kill(xvfbPid, 'SIGTERM');
|
|
232
|
+
} catch {
|
|
233
|
+
// Xvfb may have already exited.
|
|
234
|
+
}
|
|
235
|
+
clearInterval(timer);
|
|
236
|
+
}, 100);
|
|
237
|
+
`;
|
|
238
|
+
function scheduleXvfbStopAfterProcessExit(instance, parentPid = process.pid) {
|
|
239
|
+
const xvfbPid = instance.process.pid;
|
|
240
|
+
if (!xvfbPid) throw new Error('Cannot schedule Xvfb cleanup before its process starts');
|
|
241
|
+
const monitor = (0, external_node_child_process_namespaceObject.spawn)(process.execPath, [
|
|
242
|
+
'-e',
|
|
243
|
+
xvfbCleanupMonitorScript,
|
|
244
|
+
String(parentPid),
|
|
245
|
+
String(xvfbPid)
|
|
246
|
+
], {
|
|
247
|
+
detached: true,
|
|
248
|
+
stdio: 'ignore'
|
|
249
|
+
});
|
|
250
|
+
monitor.on('error', (error)=>{
|
|
251
|
+
debugXvfb(`Xvfb cleanup monitor failed: ${error.message}`);
|
|
252
|
+
});
|
|
253
|
+
instance.process.unref();
|
|
254
|
+
monitor.unref();
|
|
255
|
+
return monitor;
|
|
256
|
+
}
|
|
257
|
+
function createXvfbSignalCleanup(cleanup, source = process) {
|
|
345
258
|
return ()=>{
|
|
346
259
|
if (!(0, interrupt_namespaceObject.hasActiveCliInterruptWaiter)(source)) cleanup();
|
|
347
260
|
};
|
|
@@ -574,33 +487,19 @@ function runPowershell(script) {
|
|
|
574
487
|
windowsHide: true
|
|
575
488
|
});
|
|
576
489
|
}
|
|
577
|
-
function
|
|
490
|
+
function listWindowsDisplays() {
|
|
578
491
|
const script = `
|
|
579
492
|
Add-Type -AssemblyName System.Windows.Forms
|
|
580
493
|
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
581
|
-
$
|
|
582
|
-
[PSCustomObject]@{
|
|
583
|
-
id = $_.DeviceName
|
|
584
|
-
name = $_.DeviceName
|
|
585
|
-
primary = $_.Primary
|
|
586
|
-
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
587
|
-
}
|
|
494
|
+
[PSCustomObject]@{ id = $_.DeviceName; name = $_.DeviceName; primary = $_.Primary }
|
|
588
495
|
}
|
|
589
496
|
ConvertTo-Json @($s) -Compress
|
|
590
497
|
`.trim();
|
|
591
|
-
const
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
597
|
-
return displays;
|
|
598
|
-
}
|
|
599
|
-
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
600
|
-
return geometries.map((display)=>({
|
|
601
|
-
id: display.id,
|
|
602
|
-
name: display.name,
|
|
603
|
-
primary: display.primary
|
|
498
|
+
const parsed = JSON.parse(runPowershell(script).trim());
|
|
499
|
+
return parsed.map((d)=>({
|
|
500
|
+
id: String(d.id),
|
|
501
|
+
name: d.name || String(d.id),
|
|
502
|
+
primary: d.primary || false
|
|
604
503
|
}));
|
|
605
504
|
}
|
|
606
505
|
let device_libnut = null;
|
|
@@ -702,16 +601,6 @@ function getDisplayInfoBinary() {
|
|
|
702
601
|
function isFiniteNumber(value) {
|
|
703
602
|
return 'number' == typeof value && Number.isFinite(value);
|
|
704
603
|
}
|
|
705
|
-
function isDisplayBounds(value) {
|
|
706
|
-
if (!value || 'object' != typeof value) return false;
|
|
707
|
-
const bounds = value;
|
|
708
|
-
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
709
|
-
}
|
|
710
|
-
function isWindowsDisplayGeometry(value) {
|
|
711
|
-
if (!value || 'object' != typeof value) return false;
|
|
712
|
-
const candidate = value;
|
|
713
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
714
|
-
}
|
|
715
604
|
function isDarwinDisplayGeometry(value) {
|
|
716
605
|
if (!value || 'object' != typeof value) return false;
|
|
717
606
|
const candidate = value;
|
|
@@ -755,11 +644,8 @@ function readDarwinFrontmostApplication() {
|
|
|
755
644
|
}
|
|
756
645
|
}
|
|
757
646
|
async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
|
|
647
|
+
await inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
758
648
|
const current = inputDriver.getMousePos();
|
|
759
|
-
const drift = {
|
|
760
|
-
x: current.x - targetX,
|
|
761
|
-
y: current.y - targetY
|
|
762
|
-
};
|
|
763
649
|
debugComputerInput('tap mouse moved %o', {
|
|
764
650
|
reason,
|
|
765
651
|
target: {
|
|
@@ -767,7 +653,10 @@ async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDurati
|
|
|
767
653
|
y: targetY
|
|
768
654
|
},
|
|
769
655
|
current,
|
|
770
|
-
drift
|
|
656
|
+
drift: {
|
|
657
|
+
x: current.x - targetX,
|
|
658
|
+
y: current.y - targetY
|
|
659
|
+
}
|
|
771
660
|
});
|
|
772
661
|
await inputDriver.withMouseButton('left', async ()=>{
|
|
773
662
|
debugComputerInput('tap mouse down %o', {
|
|
@@ -786,18 +675,9 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
786
675
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
787
676
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
788
677
|
}
|
|
789
|
-
function
|
|
790
|
-
if (
|
|
791
|
-
|
|
792
|
-
return displays.find((display)=>display.id === displayId);
|
|
793
|
-
}
|
|
794
|
-
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
795
|
-
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
796
|
-
if ('win32' === process.platform) {
|
|
797
|
-
const geometry = resolveWindowsDisplayGeometryFromList(displayId, windowsDisplays ?? readWindowsDisplayGeometries());
|
|
798
|
-
if (!geometry) throw new Error(displayId ? `Requested Windows display not found: ${displayId}` : 'No Windows displays were detected');
|
|
799
|
-
return geometry;
|
|
800
|
-
}
|
|
678
|
+
function resolveDisplayGeometry(displayId) {
|
|
679
|
+
if ('darwin' !== process.platform) return;
|
|
680
|
+
return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
801
681
|
}
|
|
802
682
|
function mapDisplayLocalPointToGlobal(point, geometry) {
|
|
803
683
|
if (!geometry) return point;
|
|
@@ -883,20 +763,6 @@ function normalizePrimaryKey(key) {
|
|
|
883
763
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
884
764
|
}
|
|
885
765
|
class ComputerDevice {
|
|
886
|
-
async moveGlobalPointer(point, context, smooth) {
|
|
887
|
-
const target = {
|
|
888
|
-
x: Math.round(point.x),
|
|
889
|
-
y: Math.round(point.y)
|
|
890
|
-
};
|
|
891
|
-
if (smooth) await this.inputDriver.smoothMoveMouse(target.x, target.y, smooth.smoothSteps, smooth.smoothDelay);
|
|
892
|
-
else this.inputDriver.moveMouse(target.x, target.y);
|
|
893
|
-
await this.inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
894
|
-
if ('win32' === process.platform) await this.inputDriver.correctMousePosition(target.x, target.y, context);
|
|
895
|
-
return target;
|
|
896
|
-
}
|
|
897
|
-
moveDisplayPointer(point, context, smooth) {
|
|
898
|
-
return this.moveGlobalPointer(this.toGlobalPoint(point), context, smooth);
|
|
899
|
-
}
|
|
900
766
|
async focusKeyboardTarget(element, delayMs) {
|
|
901
767
|
const [x, y] = element.center;
|
|
902
768
|
if ('darwin' === process.platform) await this.inputPrimitives.pointer.tap({
|
|
@@ -904,10 +770,11 @@ class ComputerDevice {
|
|
|
904
770
|
y
|
|
905
771
|
});
|
|
906
772
|
else {
|
|
907
|
-
|
|
773
|
+
const point = this.toGlobalPoint({
|
|
908
774
|
x,
|
|
909
775
|
y
|
|
910
|
-
}
|
|
776
|
+
});
|
|
777
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
911
778
|
this.inputDriver.mouseClick('left');
|
|
912
779
|
}
|
|
913
780
|
await this.inputDriver.delay(delayMs);
|
|
@@ -926,7 +793,7 @@ class ComputerDevice {
|
|
|
926
793
|
}));
|
|
927
794
|
} catch (error) {
|
|
928
795
|
debugDevice(`Failed to list displays: ${error}`);
|
|
929
|
-
|
|
796
|
+
return [];
|
|
930
797
|
}
|
|
931
798
|
}
|
|
932
799
|
async connect() {
|
|
@@ -938,24 +805,26 @@ class ComputerDevice {
|
|
|
938
805
|
this.xvfbInstance = await startXvfb({
|
|
939
806
|
resolution: this.options?.xvfbResolution
|
|
940
807
|
});
|
|
808
|
+
if (this.options?.keepXvfbAliveUntilProcessExit) scheduleXvfbStopAfterProcessExit(this.xvfbInstance);
|
|
941
809
|
process.env.DISPLAY = this.xvfbInstance.display;
|
|
942
810
|
debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
|
|
943
|
-
this.
|
|
944
|
-
|
|
945
|
-
this.xvfbInstance
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
811
|
+
if (!this.options?.keepXvfbAliveUntilProcessExit) {
|
|
812
|
+
this.xvfbCleanup = ()=>{
|
|
813
|
+
if (this.xvfbInstance) {
|
|
814
|
+
this.xvfbInstance.stop();
|
|
815
|
+
this.xvfbInstance = void 0;
|
|
816
|
+
}
|
|
817
|
+
};
|
|
818
|
+
this.xvfbSignalCleanup = createXvfbSignalCleanup(()=>this.xvfbCleanup?.());
|
|
819
|
+
process.on('exit', this.xvfbCleanup);
|
|
820
|
+
process.on('SIGINT', this.xvfbSignalCleanup);
|
|
821
|
+
process.on('SIGTERM', this.xvfbSignalCleanup);
|
|
822
|
+
}
|
|
953
823
|
}
|
|
954
824
|
device_libnut = await getLibnut();
|
|
955
|
-
|
|
956
|
-
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
825
|
+
this.displayGeometry = resolveDisplayGeometry(this.displayId);
|
|
957
826
|
const size = await this.size();
|
|
958
|
-
const displays =
|
|
827
|
+
const displays = await ComputerDevice.listDisplays();
|
|
959
828
|
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
960
829
|
this.description = `
|
|
961
830
|
Type: Computer
|
|
@@ -965,28 +834,28 @@ Screen Size: ${size.width}x${size.height}
|
|
|
965
834
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
966
835
|
`;
|
|
967
836
|
debugDevice('Computer device connected', this.description);
|
|
968
|
-
await this.healthCheck(
|
|
837
|
+
await this.healthCheck();
|
|
969
838
|
} catch (error) {
|
|
970
839
|
if (this.xvfbInstance) {
|
|
971
|
-
this.xvfbInstance.stop();
|
|
840
|
+
if (!this.options?.keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
972
841
|
this.xvfbInstance = void 0;
|
|
973
842
|
}
|
|
974
843
|
if (this.xvfbCleanup) {
|
|
975
844
|
process.removeListener('exit', this.xvfbCleanup);
|
|
976
|
-
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
977
845
|
this.xvfbCleanup = void 0;
|
|
978
846
|
}
|
|
979
|
-
if (this.
|
|
980
|
-
process.removeListener('SIGINT', this.
|
|
981
|
-
this.
|
|
847
|
+
if (this.xvfbSignalCleanup) {
|
|
848
|
+
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
849
|
+
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
850
|
+
this.xvfbSignalCleanup = void 0;
|
|
982
851
|
}
|
|
983
852
|
debugDevice(`Failed to connect: ${error}`);
|
|
984
853
|
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
985
854
|
}
|
|
986
855
|
}
|
|
987
|
-
async healthCheck(
|
|
856
|
+
async healthCheck() {
|
|
988
857
|
console.log('[HealthCheck] Starting health check...');
|
|
989
|
-
console.log("[HealthCheck] @midscene/computer v1.12.2
|
|
858
|
+
console.log("[HealthCheck] @midscene/computer v1.12.2");
|
|
990
859
|
console.log('[HealthCheck] Taking screenshot...');
|
|
991
860
|
const screenshotTimeout = 15000;
|
|
992
861
|
let timeoutId;
|
|
@@ -998,27 +867,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
998
867
|
timeoutPromise
|
|
999
868
|
]);
|
|
1000
869
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1001
|
-
console.log('[HealthCheck]
|
|
870
|
+
console.log('[HealthCheck] Moving mouse...');
|
|
1002
871
|
const startPos = this.inputDriver.getMousePos();
|
|
1003
872
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
} finally{
|
|
1019
|
-
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
1020
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
1021
|
-
}
|
|
873
|
+
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
874
|
+
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
875
|
+
const targetX = startPos.x + offsetX;
|
|
876
|
+
const targetY = startPos.y + offsetY;
|
|
877
|
+
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
878
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
879
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
880
|
+
const movedPos = this.inputDriver.getMousePos();
|
|
881
|
+
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
882
|
+
const deltaX = Math.abs(movedPos.x - targetX);
|
|
883
|
+
const deltaY = Math.abs(movedPos.y - targetY);
|
|
884
|
+
if (deltaX > 5 || deltaY > 5) {
|
|
885
|
+
const msg = `[HealthCheck] WARNING: Mouse control may not be working. Expected (${targetX}, ${targetY}), got (${movedPos.x}, ${movedPos.y}), delta=(${deltaX}, ${deltaY})`;
|
|
886
|
+
warnDevice(msg);
|
|
1022
887
|
}
|
|
1023
888
|
if ('win32' === process.platform && !this.isRunningAsAdmin()) {
|
|
1024
889
|
const hint = [
|
|
@@ -1028,7 +893,10 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1028
893
|
].join(' ');
|
|
1029
894
|
warnDevice(`[HealthCheck] ${hint}`);
|
|
1030
895
|
}
|
|
896
|
+
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
897
|
+
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
1031
898
|
console.log('[HealthCheck] Listing monitors...');
|
|
899
|
+
const displays = await ComputerDevice.listDisplays();
|
|
1032
900
|
if (displays.length > 0) {
|
|
1033
901
|
console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
|
|
1034
902
|
for (const display of displays){
|
|
@@ -1211,14 +1079,10 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1211
1079
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1212
1080
|
if ('win32' === process.platform) {
|
|
1213
1081
|
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
1214
|
-
if (activeWindowRect) {
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
};
|
|
1219
|
-
const bounds = this.displayGeometry?.bounds;
|
|
1220
|
-
if (!bounds || activeWindowCenter.x >= bounds.x && activeWindowCenter.x < bounds.x + bounds.width && activeWindowCenter.y >= bounds.y && activeWindowCenter.y < bounds.y + bounds.height) return activeWindowCenter;
|
|
1221
|
-
}
|
|
1082
|
+
if (activeWindowRect) return {
|
|
1083
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
1084
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
1085
|
+
};
|
|
1222
1086
|
}
|
|
1223
1087
|
return this.toGlobalPoint({
|
|
1224
1088
|
x: screenSize.width / 2,
|
|
@@ -1229,16 +1093,17 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1229
1093
|
if (param.locate) {
|
|
1230
1094
|
const element = param.locate;
|
|
1231
1095
|
const [x, y] = element.center;
|
|
1232
|
-
|
|
1096
|
+
const point = this.toGlobalPoint({
|
|
1233
1097
|
x,
|
|
1234
1098
|
y
|
|
1235
|
-
}
|
|
1099
|
+
});
|
|
1100
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1236
1101
|
return;
|
|
1237
1102
|
}
|
|
1238
1103
|
const screenSize = await this.size();
|
|
1239
1104
|
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
1240
1105
|
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
1241
|
-
|
|
1106
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1242
1107
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1243
1108
|
return screenSize;
|
|
1244
1109
|
}
|
|
@@ -1327,18 +1192,19 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1327
1192
|
if (this.destroyed) return;
|
|
1328
1193
|
this.destroyed = true;
|
|
1329
1194
|
this.inputDriver.destroy();
|
|
1195
|
+
const keepXvfbAliveUntilProcessExit = this.options?.keepXvfbAliveUntilProcessExit === true;
|
|
1330
1196
|
if (this.xvfbInstance) {
|
|
1331
|
-
this.xvfbInstance.stop();
|
|
1197
|
+
if (!keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
1332
1198
|
this.xvfbInstance = void 0;
|
|
1333
1199
|
}
|
|
1334
|
-
if (this.xvfbCleanup) {
|
|
1200
|
+
if (this.xvfbCleanup && !keepXvfbAliveUntilProcessExit) {
|
|
1335
1201
|
process.removeListener('exit', this.xvfbCleanup);
|
|
1336
|
-
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
1337
1202
|
this.xvfbCleanup = void 0;
|
|
1338
1203
|
}
|
|
1339
|
-
if (this.
|
|
1340
|
-
process.removeListener('SIGINT', this.
|
|
1341
|
-
this.
|
|
1204
|
+
if (this.xvfbSignalCleanup) {
|
|
1205
|
+
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
1206
|
+
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
1207
|
+
this.xvfbSignalCleanup = void 0;
|
|
1342
1208
|
}
|
|
1343
1209
|
debugDevice('Computer device destroyed');
|
|
1344
1210
|
}
|
|
@@ -1354,7 +1220,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1354
1220
|
device_define_property(this, "destroyed", false);
|
|
1355
1221
|
device_define_property(this, "xvfbInstance", void 0);
|
|
1356
1222
|
device_define_property(this, "xvfbCleanup", void 0);
|
|
1357
|
-
device_define_property(this, "
|
|
1223
|
+
device_define_property(this, "xvfbSignalCleanup", void 0);
|
|
1358
1224
|
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
1359
1225
|
getLibnut: ()=>device_libnut,
|
|
1360
1226
|
useAppleScript: ()=>this.useAppleScript,
|
|
@@ -1386,16 +1252,14 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1386
1252
|
},
|
|
1387
1253
|
holdDuration,
|
|
1388
1254
|
displayId: this.displayId,
|
|
1389
|
-
displayGeometry: this.displayGeometry
|
|
1255
|
+
displayGeometry: this.displayGeometry ? {
|
|
1256
|
+
screenIndex: this.displayGeometry.screenIndex,
|
|
1257
|
+
cgDisplayId: this.displayGeometry.cgDisplayId,
|
|
1258
|
+
bounds: this.displayGeometry.bounds
|
|
1259
|
+
} : void 0
|
|
1390
1260
|
});
|
|
1391
1261
|
const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
|
|
1392
|
-
await this.
|
|
1393
|
-
x: targetX,
|
|
1394
|
-
y: targetY
|
|
1395
|
-
}, 'Mouse did not reach the tap target', {
|
|
1396
|
-
smoothSteps: SMOOTH_MOVE_STEPS_TAP,
|
|
1397
|
-
smoothDelay: SMOOTH_MOVE_DELAY_TAP
|
|
1398
|
-
});
|
|
1262
|
+
await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
1399
1263
|
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'primary');
|
|
1400
1264
|
if (frontmostBefore && 'darwin' === process.platform) {
|
|
1401
1265
|
await (0, utils_namespaceObject.sleep)(CLICK_FOCUS_SETTLE_DELAY);
|
|
@@ -1407,43 +1271,42 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1407
1271
|
focusChanged
|
|
1408
1272
|
});
|
|
1409
1273
|
if (focusChanged) {
|
|
1410
|
-
|
|
1411
|
-
x: targetX,
|
|
1412
|
-
y: targetY
|
|
1413
|
-
}, 'Mouse did not reach the focus follow-up target');
|
|
1274
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
1414
1275
|
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'focus-follow-up');
|
|
1415
1276
|
}
|
|
1416
1277
|
}
|
|
1417
1278
|
},
|
|
1418
1279
|
doubleClick: async ({ x, y })=>{
|
|
1419
|
-
|
|
1280
|
+
const target = this.toGlobalPoint({
|
|
1420
1281
|
x,
|
|
1421
1282
|
y
|
|
1422
|
-
}
|
|
1283
|
+
});
|
|
1284
|
+
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1423
1285
|
this.inputDriver.mouseClick('left', true);
|
|
1424
1286
|
},
|
|
1425
1287
|
rightClick: async ({ x, y })=>{
|
|
1426
|
-
|
|
1288
|
+
const target = this.toGlobalPoint({
|
|
1427
1289
|
x,
|
|
1428
1290
|
y
|
|
1429
|
-
}
|
|
1291
|
+
});
|
|
1292
|
+
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1430
1293
|
this.inputDriver.mouseClick('right');
|
|
1431
1294
|
},
|
|
1432
1295
|
hover: async ({ x, y })=>{
|
|
1433
|
-
|
|
1296
|
+
const target = this.toGlobalPoint({
|
|
1434
1297
|
x,
|
|
1435
1298
|
y
|
|
1436
|
-
}, 'Mouse did not reach the hover target', {
|
|
1437
|
-
smoothSteps: SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1438
|
-
smoothDelay: SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
1439
1299
|
});
|
|
1300
|
+
await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
1440
1301
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1441
1302
|
},
|
|
1442
1303
|
dragAndDrop: async (from, to)=>{
|
|
1443
|
-
|
|
1304
|
+
const globalFrom = this.toGlobalPoint(from);
|
|
1305
|
+
const globalTo = this.toGlobalPoint(to);
|
|
1306
|
+
this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
|
|
1444
1307
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1445
1308
|
await this.inputDriver.delay(100);
|
|
1446
|
-
|
|
1309
|
+
this.inputDriver.moveMouse(Math.round(globalTo.x), Math.round(globalTo.y));
|
|
1447
1310
|
await this.inputDriver.delay(100);
|
|
1448
1311
|
});
|
|
1449
1312
|
}
|
|
@@ -2174,7 +2037,8 @@ function createLocalComputerDevice(opts) {
|
|
|
2174
2037
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2175
2038
|
keyboardDriver: opts?.keyboardDriver,
|
|
2176
2039
|
headless: opts?.headless,
|
|
2177
|
-
xvfbResolution: opts?.xvfbResolution
|
|
2040
|
+
xvfbResolution: opts?.xvfbResolution,
|
|
2041
|
+
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
2178
2042
|
});
|
|
2179
2043
|
}
|
|
2180
2044
|
function createRDPComputerDevice(opts) {
|
|
@@ -2317,6 +2181,9 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2317
2181
|
...void 0 !== keyboardTypeDelay ? {
|
|
2318
2182
|
keyboardTypeDelay
|
|
2319
2183
|
} : {},
|
|
2184
|
+
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2185
|
+
keepXvfbAliveUntilProcessExit: true
|
|
2186
|
+
} : {},
|
|
2320
2187
|
...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(opts) ?? {},
|
|
2321
2188
|
...reportOptions ?? {}
|
|
2322
2189
|
};
|
|
@@ -2382,21 +2249,23 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2382
2249
|
}
|
|
2383
2250
|
];
|
|
2384
2251
|
}
|
|
2385
|
-
constructor(
|
|
2386
|
-
super(
|
|
2252
|
+
constructor(options = {}){
|
|
2253
|
+
super(), agent_tools_define_property(this, "options", void 0), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", void 0), this.options = options, this.initArgSpec = {
|
|
2387
2254
|
namespace: 'computer',
|
|
2388
2255
|
shape: computerInitArgShape,
|
|
2389
2256
|
cli: {
|
|
2390
2257
|
preferBareKeys: true
|
|
2391
2258
|
},
|
|
2392
2259
|
adapt: (extracted)=>adaptComputerInitArgs(extracted)
|
|
2393
|
-
}
|
|
2260
|
+
};
|
|
2394
2261
|
}
|
|
2395
2262
|
}
|
|
2396
|
-
const tools = new ComputerMidsceneTools(
|
|
2263
|
+
const tools = new ComputerMidsceneTools({
|
|
2264
|
+
keepXvfbAliveUntilProcessExit: true
|
|
2265
|
+
});
|
|
2397
2266
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
2398
2267
|
stripPrefix: 'computer_',
|
|
2399
|
-
version: "1.12.2
|
|
2268
|
+
version: "1.12.2",
|
|
2400
2269
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
2401
2270
|
}).catch((e)=>{
|
|
2402
2271
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|