@midscene/computer 1.12.2-beta-20260828072235.0 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +152 -283
- package/dist/es/index.mjs +149 -282
- package/dist/lib/cli.js +152 -283
- package/dist/lib/index.js +149 -282
- package/dist/types/index.d.ts +20 -6
- package/package.json +3 -3
package/dist/lib/index.js
CHANGED
|
@@ -70,57 +70,6 @@ const external_screenshot_desktop_namespaceObject = require("screenshot-desktop"
|
|
|
70
70
|
var external_screenshot_desktop_default = /*#__PURE__*/ __webpack_require__.n(external_screenshot_desktop_namespaceObject);
|
|
71
71
|
const external_node_assert_namespaceObject = require("node:assert");
|
|
72
72
|
var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
|
|
73
|
-
const MOUSE_COORDINATE_TOLERANCE_PX = 5;
|
|
74
|
-
const MIN_CALIBRATION_AXIS_DELTA = 20;
|
|
75
|
-
const MIN_VALID_CALIBRATION_SCALE = 0.1;
|
|
76
|
-
const MAX_VALID_CALIBRATION_SCALE = 10;
|
|
77
|
-
function assertValidMouseCalibrationBounds(bounds) {
|
|
78
|
-
if (!Number.isFinite(bounds.x) || !Number.isFinite(bounds.y) || !Number.isFinite(bounds.width) || !Number.isFinite(bounds.height) || bounds.width <= 0 || bounds.height <= 0) throw new Error(`Mouse calibration bounds are invalid: (${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`);
|
|
79
|
-
}
|
|
80
|
-
function mouseCalibrationPoint(bounds, ratio) {
|
|
81
|
-
return {
|
|
82
|
-
x: Math.round(bounds.x + bounds.width * ratio),
|
|
83
|
-
y: Math.round(bounds.y + bounds.height * ratio)
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
function calculateMouseCoordinateCalibration(firstInput, firstActual, secondInput, secondActual) {
|
|
87
|
-
const inputDeltaX = secondInput.x - firstInput.x;
|
|
88
|
-
const inputDeltaY = secondInput.y - firstInput.y;
|
|
89
|
-
if (Math.abs(inputDeltaX) < MIN_CALIBRATION_AXIS_DELTA || Math.abs(inputDeltaY) < MIN_CALIBRATION_AXIS_DELTA) throw new Error(`Mouse calibration points are too close: delta=(${inputDeltaX}, ${inputDeltaY})`);
|
|
90
|
-
const scaleX = (secondActual.x - firstActual.x) / inputDeltaX;
|
|
91
|
-
const scaleY = (secondActual.y - firstActual.y) / inputDeltaY;
|
|
92
|
-
if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY) || scaleX < MIN_VALID_CALIBRATION_SCALE || scaleY < MIN_VALID_CALIBRATION_SCALE || scaleX > MAX_VALID_CALIBRATION_SCALE || scaleY > MAX_VALID_CALIBRATION_SCALE) throw new Error(`Mouse calibration produced invalid scale: (${scaleX}, ${scaleY})`);
|
|
93
|
-
return {
|
|
94
|
-
scaleX,
|
|
95
|
-
scaleY,
|
|
96
|
-
offsetX: firstActual.x - scaleX * firstInput.x,
|
|
97
|
-
offsetY: firstActual.y - scaleY * firstInput.y
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
function applyMouseCoordinateCalibration(point, calibration) {
|
|
101
|
-
return {
|
|
102
|
-
x: Math.round((point.x - calibration.offsetX) / calibration.scaleX),
|
|
103
|
-
y: Math.round((point.y - calibration.offsetY) / calibration.scaleY)
|
|
104
|
-
};
|
|
105
|
-
}
|
|
106
|
-
function mouseCoordinateCalibrationNeedsCorrection(calibration) {
|
|
107
|
-
return Math.abs(calibration.scaleX - 1) > 0.01 || Math.abs(calibration.scaleY - 1) > 0.01 || Math.abs(calibration.offsetX) > MOUSE_COORDINATE_TOLERANCE_PX || Math.abs(calibration.offsetY) > MOUSE_COORDINATE_TOLERANCE_PX;
|
|
108
|
-
}
|
|
109
|
-
function getMouseCoordinateDrift(expected, actual) {
|
|
110
|
-
return {
|
|
111
|
-
x: actual.x - expected.x,
|
|
112
|
-
y: actual.y - expected.y
|
|
113
|
-
};
|
|
114
|
-
}
|
|
115
|
-
function mouseCoordinateDriftIsWithinTolerance(drift) {
|
|
116
|
-
return Math.abs(drift.x) <= MOUSE_COORDINATE_TOLERANCE_PX && Math.abs(drift.y) <= MOUSE_COORDINATE_TOLERANCE_PX;
|
|
117
|
-
}
|
|
118
|
-
function mouseCoordinateCorrectionPoint(requested, drift) {
|
|
119
|
-
return {
|
|
120
|
-
x: requested.x - drift.x,
|
|
121
|
-
y: requested.y - drift.y
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
73
|
function _define_property(obj, key, value) {
|
|
125
74
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
126
75
|
value: value,
|
|
@@ -131,9 +80,6 @@ function _define_property(obj, key, value) {
|
|
|
131
80
|
else obj[key] = value;
|
|
132
81
|
return obj;
|
|
133
82
|
}
|
|
134
|
-
const CALIBRATION_SETTLE_DELAY_MS = 80;
|
|
135
|
-
const MOUSE_CORRECTION_SETTLE_DELAY_MS = 50;
|
|
136
|
-
const MAX_MOUSE_CORRECTION_ATTEMPTS = 3;
|
|
137
83
|
class ComputerInputDriver {
|
|
138
84
|
destroy() {
|
|
139
85
|
if (this.destroyed) return;
|
|
@@ -147,82 +93,7 @@ class ComputerInputDriver {
|
|
|
147
93
|
return this.getLibnutOrThrow('getMousePos').getMousePos();
|
|
148
94
|
}
|
|
149
95
|
moveMouse(x, y) {
|
|
150
|
-
|
|
151
|
-
x,
|
|
152
|
-
y
|
|
153
|
-
}, this.mouseCoordinateCalibration) : {
|
|
154
|
-
x,
|
|
155
|
-
y
|
|
156
|
-
};
|
|
157
|
-
this.getLibnutOrThrow('moveMouse').moveMouse(target.x, target.y);
|
|
158
|
-
}
|
|
159
|
-
async calibrateMouseCoordinates(bounds) {
|
|
160
|
-
this.assertActive('calibrateMouseCoordinates');
|
|
161
|
-
assertValidMouseCalibrationBounds(bounds);
|
|
162
|
-
const firstInput = mouseCalibrationPoint(bounds, 0.1);
|
|
163
|
-
const secondInput = mouseCalibrationPoint(bounds, 0.3);
|
|
164
|
-
const savedPosition = this.getMousePos();
|
|
165
|
-
this.mouseCoordinateCalibration = void 0;
|
|
166
|
-
try {
|
|
167
|
-
this.moveMouse(firstInput.x, firstInput.y);
|
|
168
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
169
|
-
const firstActual = this.getMousePos();
|
|
170
|
-
this.moveMouse(secondInput.x, secondInput.y);
|
|
171
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
172
|
-
const secondActual = this.getMousePos();
|
|
173
|
-
const calibration = calculateMouseCoordinateCalibration(firstInput, firstActual, secondInput, secondActual);
|
|
174
|
-
const needsCorrection = mouseCoordinateCalibrationNeedsCorrection(calibration);
|
|
175
|
-
this.mouseCoordinateCalibration = needsCorrection ? calibration : void 0;
|
|
176
|
-
let verificationDrift = {
|
|
177
|
-
x: 0,
|
|
178
|
-
y: 0
|
|
179
|
-
};
|
|
180
|
-
if (needsCorrection) {
|
|
181
|
-
const verificationTarget = mouseCalibrationPoint(bounds, 0.5);
|
|
182
|
-
this.moveMouse(verificationTarget.x, verificationTarget.y);
|
|
183
|
-
await this.delay(CALIBRATION_SETTLE_DELAY_MS);
|
|
184
|
-
verificationDrift = this.assertMousePosition(verificationTarget.x, verificationTarget.y, 'Mouse coordinate calibration verification');
|
|
185
|
-
}
|
|
186
|
-
this.options.debug(needsCorrection ? `Mouse coordinate calibration applied: scale=(${calibration.scaleX.toFixed(4)}, ${calibration.scaleY.toFixed(4)}), offset=(${calibration.offsetX.toFixed(1)}, ${calibration.offsetY.toFixed(1)}), verification drift=(${verificationDrift.x}, ${verificationDrift.y})` : 'Mouse coordinate calibration is identity');
|
|
187
|
-
this.moveMouse(savedPosition.x, savedPosition.y);
|
|
188
|
-
} catch (error) {
|
|
189
|
-
try {
|
|
190
|
-
this.moveMouse(savedPosition.x, savedPosition.y);
|
|
191
|
-
} finally{
|
|
192
|
-
this.mouseCoordinateCalibration = void 0;
|
|
193
|
-
}
|
|
194
|
-
throw error;
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
assertMousePosition(targetX, targetY, context) {
|
|
198
|
-
const current = this.getMousePos();
|
|
199
|
-
const drift = getMouseCoordinateDrift({
|
|
200
|
-
x: targetX,
|
|
201
|
-
y: targetY
|
|
202
|
-
}, current);
|
|
203
|
-
if (!mouseCoordinateDriftIsWithinTolerance(drift)) throw new Error(`${context}: expected (${targetX}, ${targetY}), got (${current.x}, ${current.y}), drift=(${drift.x}, ${drift.y})`);
|
|
204
|
-
return drift;
|
|
205
|
-
}
|
|
206
|
-
async correctMousePosition(targetX, targetY, context) {
|
|
207
|
-
const target = {
|
|
208
|
-
x: targetX,
|
|
209
|
-
y: targetY
|
|
210
|
-
};
|
|
211
|
-
let requested = target;
|
|
212
|
-
for(let attempt = 0; attempt <= MAX_MOUSE_CORRECTION_ATTEMPTS; attempt++){
|
|
213
|
-
const current = this.getMousePos();
|
|
214
|
-
const drift = getMouseCoordinateDrift(target, current);
|
|
215
|
-
if (mouseCoordinateDriftIsWithinTolerance(drift)) {
|
|
216
|
-
if (attempt > 0) this.options.debug(`${context}: pointer correction converged after ${attempt} attempt(s), drift=(${drift.x}, ${drift.y})`);
|
|
217
|
-
return drift;
|
|
218
|
-
}
|
|
219
|
-
if (attempt === MAX_MOUSE_CORRECTION_ATTEMPTS) return this.assertMousePosition(targetX, targetY, context);
|
|
220
|
-
requested = mouseCoordinateCorrectionPoint(requested, drift);
|
|
221
|
-
this.options.debug(`${context}: correcting pointer drift=(${drift.x}, ${drift.y}) with requested point=(${requested.x}, ${requested.y}), attempt=${attempt + 1}/${MAX_MOUSE_CORRECTION_ATTEMPTS}`);
|
|
222
|
-
this.moveMouse(requested.x, requested.y);
|
|
223
|
-
await this.delay(MOUSE_CORRECTION_SETTLE_DELAY_MS);
|
|
224
|
-
}
|
|
225
|
-
throw new Error(`${context}: mouse correction ended unexpectedly`);
|
|
96
|
+
this.getLibnutOrThrow('moveMouse').moveMouse(x, y);
|
|
226
97
|
}
|
|
227
98
|
focusActiveWindow() {
|
|
228
99
|
const lib = this.getLibnutOrThrow('focusActiveWindow');
|
|
@@ -358,7 +229,6 @@ class ComputerInputDriver {
|
|
|
358
229
|
constructor(options){
|
|
359
230
|
_define_property(this, "options", void 0);
|
|
360
231
|
_define_property(this, "destroyed", void 0);
|
|
361
|
-
_define_property(this, "mouseCoordinateCalibration", void 0);
|
|
362
232
|
_define_property(this, "pendingInputDelayWaits", void 0);
|
|
363
233
|
this.options = options;
|
|
364
234
|
this.destroyed = false;
|
|
@@ -367,7 +237,50 @@ class ComputerInputDriver {
|
|
|
367
237
|
}
|
|
368
238
|
const interrupt_namespaceObject = require("@midscene/shared/cli/interrupt");
|
|
369
239
|
const debugXvfb = (0, logger_namespaceObject.getDebug)('computer:xvfb');
|
|
370
|
-
|
|
240
|
+
const xvfbCleanupMonitorScript = String.raw`
|
|
241
|
+
const parentPid = Number(process.argv[1]);
|
|
242
|
+
const xvfbPid = Number(process.argv[2]);
|
|
243
|
+
const timer = setInterval(() => {
|
|
244
|
+
try {
|
|
245
|
+
process.kill(xvfbPid, 0);
|
|
246
|
+
} catch {
|
|
247
|
+
clearInterval(timer);
|
|
248
|
+
process.exit(0);
|
|
249
|
+
}
|
|
250
|
+
try {
|
|
251
|
+
process.kill(parentPid, 0);
|
|
252
|
+
return;
|
|
253
|
+
} catch {
|
|
254
|
+
// The owner is gone, so its X11 clients can no longer receive XIO errors.
|
|
255
|
+
}
|
|
256
|
+
try {
|
|
257
|
+
process.kill(xvfbPid, 'SIGTERM');
|
|
258
|
+
} catch {
|
|
259
|
+
// Xvfb may have already exited.
|
|
260
|
+
}
|
|
261
|
+
clearInterval(timer);
|
|
262
|
+
}, 100);
|
|
263
|
+
`;
|
|
264
|
+
function scheduleXvfbStopAfterProcessExit(instance, parentPid = process.pid) {
|
|
265
|
+
const xvfbPid = instance.process.pid;
|
|
266
|
+
if (!xvfbPid) throw new Error('Cannot schedule Xvfb cleanup before its process starts');
|
|
267
|
+
const monitor = (0, external_node_child_process_namespaceObject.spawn)(process.execPath, [
|
|
268
|
+
'-e',
|
|
269
|
+
xvfbCleanupMonitorScript,
|
|
270
|
+
String(parentPid),
|
|
271
|
+
String(xvfbPid)
|
|
272
|
+
], {
|
|
273
|
+
detached: true,
|
|
274
|
+
stdio: 'ignore'
|
|
275
|
+
});
|
|
276
|
+
monitor.on('error', (error)=>{
|
|
277
|
+
debugXvfb(`Xvfb cleanup monitor failed: ${error.message}`);
|
|
278
|
+
});
|
|
279
|
+
instance.process.unref();
|
|
280
|
+
monitor.unref();
|
|
281
|
+
return monitor;
|
|
282
|
+
}
|
|
283
|
+
function createXvfbSignalCleanup(cleanup, source = process) {
|
|
371
284
|
return ()=>{
|
|
372
285
|
if (!(0, interrupt_namespaceObject.hasActiveCliInterruptWaiter)(source)) cleanup();
|
|
373
286
|
};
|
|
@@ -600,33 +513,19 @@ function runPowershell(script) {
|
|
|
600
513
|
windowsHide: true
|
|
601
514
|
});
|
|
602
515
|
}
|
|
603
|
-
function
|
|
516
|
+
function listWindowsDisplays() {
|
|
604
517
|
const script = `
|
|
605
518
|
Add-Type -AssemblyName System.Windows.Forms
|
|
606
519
|
$s = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
|
|
607
|
-
$
|
|
608
|
-
[PSCustomObject]@{
|
|
609
|
-
id = $_.DeviceName
|
|
610
|
-
name = $_.DeviceName
|
|
611
|
-
primary = $_.Primary
|
|
612
|
-
bounds = [PSCustomObject]@{ x = $b.X; y = $b.Y; width = $b.Width; height = $b.Height }
|
|
613
|
-
}
|
|
520
|
+
[PSCustomObject]@{ id = $_.DeviceName; name = $_.DeviceName; primary = $_.Primary }
|
|
614
521
|
}
|
|
615
522
|
ConvertTo-Json @($s) -Compress
|
|
616
523
|
`.trim();
|
|
617
|
-
const
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
if (displays.length !== parsed.length || 0 === displays.length) throw new Error('Windows display enumeration returned invalid geometry');
|
|
623
|
-
return displays;
|
|
624
|
-
}
|
|
625
|
-
function listWindowsDisplays(geometries = readWindowsDisplayGeometries()) {
|
|
626
|
-
return geometries.map((display)=>({
|
|
627
|
-
id: display.id,
|
|
628
|
-
name: display.name,
|
|
629
|
-
primary: display.primary
|
|
524
|
+
const parsed = JSON.parse(runPowershell(script).trim());
|
|
525
|
+
return parsed.map((d)=>({
|
|
526
|
+
id: String(d.id),
|
|
527
|
+
name: d.name || String(d.id),
|
|
528
|
+
primary: d.primary || false
|
|
630
529
|
}));
|
|
631
530
|
}
|
|
632
531
|
let device_libnut = null;
|
|
@@ -728,16 +627,6 @@ function getDisplayInfoBinary() {
|
|
|
728
627
|
function isFiniteNumber(value) {
|
|
729
628
|
return 'number' == typeof value && Number.isFinite(value);
|
|
730
629
|
}
|
|
731
|
-
function isDisplayBounds(value) {
|
|
732
|
-
if (!value || 'object' != typeof value) return false;
|
|
733
|
-
const bounds = value;
|
|
734
|
-
return isFiniteNumber(bounds.x) && isFiniteNumber(bounds.y) && isFiniteNumber(bounds.width) && isFiniteNumber(bounds.height) && bounds.width > 0 && bounds.height > 0;
|
|
735
|
-
}
|
|
736
|
-
function isWindowsDisplayGeometry(value) {
|
|
737
|
-
if (!value || 'object' != typeof value) return false;
|
|
738
|
-
const candidate = value;
|
|
739
|
-
return 'string' == typeof candidate.id && candidate.id.length > 0 && 'string' == typeof candidate.name && 'boolean' == typeof candidate.primary && isDisplayBounds(candidate.bounds);
|
|
740
|
-
}
|
|
741
630
|
function isDarwinDisplayGeometry(value) {
|
|
742
631
|
if (!value || 'object' != typeof value) return false;
|
|
743
632
|
const candidate = value;
|
|
@@ -781,11 +670,8 @@ function readDarwinFrontmostApplication() {
|
|
|
781
670
|
}
|
|
782
671
|
}
|
|
783
672
|
async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDuration, reason) {
|
|
673
|
+
await inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
784
674
|
const current = inputDriver.getMousePos();
|
|
785
|
-
const drift = {
|
|
786
|
-
x: current.x - targetX,
|
|
787
|
-
y: current.y - targetY
|
|
788
|
-
};
|
|
789
675
|
debugComputerInput('tap mouse moved %o', {
|
|
790
676
|
reason,
|
|
791
677
|
target: {
|
|
@@ -793,7 +679,10 @@ async function pressMouseAtGlobalPoint(inputDriver, targetX, targetY, holdDurati
|
|
|
793
679
|
y: targetY
|
|
794
680
|
},
|
|
795
681
|
current,
|
|
796
|
-
drift
|
|
682
|
+
drift: {
|
|
683
|
+
x: current.x - targetX,
|
|
684
|
+
y: current.y - targetY
|
|
685
|
+
}
|
|
797
686
|
});
|
|
798
687
|
await inputDriver.withMouseButton('left', async ()=>{
|
|
799
688
|
debugComputerInput('tap mouse down %o', {
|
|
@@ -812,18 +701,9 @@ function resolveDarwinDisplayGeometryFromList(displayId, displays) {
|
|
|
812
701
|
if (void 0 === displayId || '' === displayId) return displays.find((display)=>display.primary) || displays.find((display)=>0 === display.screenIndex) || displays[0];
|
|
813
702
|
return displays.find((display)=>display.screenIndex === screenIndex) || displays.find((display)=>display.cgDisplayId === screenIndex);
|
|
814
703
|
}
|
|
815
|
-
function
|
|
816
|
-
if (
|
|
817
|
-
|
|
818
|
-
return displays.find((display)=>display.id === displayId);
|
|
819
|
-
}
|
|
820
|
-
function resolveDisplayGeometry(displayId, windowsDisplays) {
|
|
821
|
-
if ('darwin' === process.platform) return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
822
|
-
if ('win32' === process.platform) {
|
|
823
|
-
const geometry = resolveWindowsDisplayGeometryFromList(displayId, windowsDisplays ?? readWindowsDisplayGeometries());
|
|
824
|
-
if (!geometry) throw new Error(displayId ? `Requested Windows display not found: ${displayId}` : 'No Windows displays were detected');
|
|
825
|
-
return geometry;
|
|
826
|
-
}
|
|
704
|
+
function resolveDisplayGeometry(displayId) {
|
|
705
|
+
if ('darwin' !== process.platform) return;
|
|
706
|
+
return resolveDarwinDisplayGeometryFromList(displayId, readDarwinDisplayGeometries());
|
|
827
707
|
}
|
|
828
708
|
function mapDisplayLocalPointToGlobal(point, geometry) {
|
|
829
709
|
if (!geometry) return point;
|
|
@@ -909,20 +789,6 @@ function normalizePrimaryKey(key) {
|
|
|
909
789
|
return KEY_NAME_MAP[lowerKey] || lowerKey;
|
|
910
790
|
}
|
|
911
791
|
class ComputerDevice {
|
|
912
|
-
async moveGlobalPointer(point, context, smooth) {
|
|
913
|
-
const target = {
|
|
914
|
-
x: Math.round(point.x),
|
|
915
|
-
y: Math.round(point.y)
|
|
916
|
-
};
|
|
917
|
-
if (smooth) await this.inputDriver.smoothMoveMouse(target.x, target.y, smooth.smoothSteps, smooth.smoothDelay);
|
|
918
|
-
else this.inputDriver.moveMouse(target.x, target.y);
|
|
919
|
-
await this.inputDriver.delay(CLICK_SETTLE_DELAY);
|
|
920
|
-
if ('win32' === process.platform) await this.inputDriver.correctMousePosition(target.x, target.y, context);
|
|
921
|
-
return target;
|
|
922
|
-
}
|
|
923
|
-
moveDisplayPointer(point, context, smooth) {
|
|
924
|
-
return this.moveGlobalPointer(this.toGlobalPoint(point), context, smooth);
|
|
925
|
-
}
|
|
926
792
|
async focusKeyboardTarget(element, delayMs) {
|
|
927
793
|
const [x, y] = element.center;
|
|
928
794
|
if ('darwin' === process.platform) await this.inputPrimitives.pointer.tap({
|
|
@@ -930,10 +796,11 @@ class ComputerDevice {
|
|
|
930
796
|
y
|
|
931
797
|
});
|
|
932
798
|
else {
|
|
933
|
-
|
|
799
|
+
const point = this.toGlobalPoint({
|
|
934
800
|
x,
|
|
935
801
|
y
|
|
936
|
-
}
|
|
802
|
+
});
|
|
803
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
937
804
|
this.inputDriver.mouseClick('left');
|
|
938
805
|
}
|
|
939
806
|
await this.inputDriver.delay(delayMs);
|
|
@@ -952,7 +819,7 @@ class ComputerDevice {
|
|
|
952
819
|
}));
|
|
953
820
|
} catch (error) {
|
|
954
821
|
debugDevice(`Failed to list displays: ${error}`);
|
|
955
|
-
|
|
822
|
+
return [];
|
|
956
823
|
}
|
|
957
824
|
}
|
|
958
825
|
async connect() {
|
|
@@ -964,24 +831,26 @@ class ComputerDevice {
|
|
|
964
831
|
this.xvfbInstance = await startXvfb({
|
|
965
832
|
resolution: this.options?.xvfbResolution
|
|
966
833
|
});
|
|
834
|
+
if (this.options?.keepXvfbAliveUntilProcessExit) scheduleXvfbStopAfterProcessExit(this.xvfbInstance);
|
|
967
835
|
process.env.DISPLAY = this.xvfbInstance.display;
|
|
968
836
|
debugDevice(`Xvfb started on display ${this.xvfbInstance.display}`);
|
|
969
|
-
this.
|
|
970
|
-
|
|
971
|
-
this.xvfbInstance
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
837
|
+
if (!this.options?.keepXvfbAliveUntilProcessExit) {
|
|
838
|
+
this.xvfbCleanup = ()=>{
|
|
839
|
+
if (this.xvfbInstance) {
|
|
840
|
+
this.xvfbInstance.stop();
|
|
841
|
+
this.xvfbInstance = void 0;
|
|
842
|
+
}
|
|
843
|
+
};
|
|
844
|
+
this.xvfbSignalCleanup = createXvfbSignalCleanup(()=>this.xvfbCleanup?.());
|
|
845
|
+
process.on('exit', this.xvfbCleanup);
|
|
846
|
+
process.on('SIGINT', this.xvfbSignalCleanup);
|
|
847
|
+
process.on('SIGTERM', this.xvfbSignalCleanup);
|
|
848
|
+
}
|
|
979
849
|
}
|
|
980
850
|
device_libnut = await getLibnut();
|
|
981
|
-
|
|
982
|
-
this.displayGeometry = resolveDisplayGeometry(this.displayId, windowsDisplayGeometries);
|
|
851
|
+
this.displayGeometry = resolveDisplayGeometry(this.displayId);
|
|
983
852
|
const size = await this.size();
|
|
984
|
-
const displays =
|
|
853
|
+
const displays = await ComputerDevice.listDisplays();
|
|
985
854
|
const headlessInfo = this.xvfbInstance ? `\nHeadless: true (Xvfb on ${this.xvfbInstance.display})` : '';
|
|
986
855
|
this.description = `
|
|
987
856
|
Type: Computer
|
|
@@ -991,28 +860,28 @@ Screen Size: ${size.width}x${size.height}
|
|
|
991
860
|
Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ') : 'Unknown'}${headlessInfo}
|
|
992
861
|
`;
|
|
993
862
|
debugDevice('Computer device connected', this.description);
|
|
994
|
-
await this.healthCheck(
|
|
863
|
+
await this.healthCheck();
|
|
995
864
|
} catch (error) {
|
|
996
865
|
if (this.xvfbInstance) {
|
|
997
|
-
this.xvfbInstance.stop();
|
|
866
|
+
if (!this.options?.keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
998
867
|
this.xvfbInstance = void 0;
|
|
999
868
|
}
|
|
1000
869
|
if (this.xvfbCleanup) {
|
|
1001
870
|
process.removeListener('exit', this.xvfbCleanup);
|
|
1002
|
-
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
1003
871
|
this.xvfbCleanup = void 0;
|
|
1004
872
|
}
|
|
1005
|
-
if (this.
|
|
1006
|
-
process.removeListener('SIGINT', this.
|
|
1007
|
-
this.
|
|
873
|
+
if (this.xvfbSignalCleanup) {
|
|
874
|
+
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
875
|
+
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
876
|
+
this.xvfbSignalCleanup = void 0;
|
|
1008
877
|
}
|
|
1009
878
|
debugDevice(`Failed to connect: ${error}`);
|
|
1010
879
|
throw new Error(`Unable to connect to computer device: ${error}`);
|
|
1011
880
|
}
|
|
1012
881
|
}
|
|
1013
|
-
async healthCheck(
|
|
882
|
+
async healthCheck() {
|
|
1014
883
|
console.log('[HealthCheck] Starting health check...');
|
|
1015
|
-
console.log("[HealthCheck] @midscene/computer v1.12.2
|
|
884
|
+
console.log("[HealthCheck] @midscene/computer v1.12.2");
|
|
1016
885
|
console.log('[HealthCheck] Taking screenshot...');
|
|
1017
886
|
const screenshotTimeout = 15000;
|
|
1018
887
|
let timeoutId;
|
|
@@ -1024,27 +893,23 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1024
893
|
timeoutPromise
|
|
1025
894
|
]);
|
|
1026
895
|
console.log(`[HealthCheck] Screenshot succeeded (length=${base64.length})`);
|
|
1027
|
-
console.log('[HealthCheck]
|
|
896
|
+
console.log('[HealthCheck] Moving mouse...');
|
|
1028
897
|
const startPos = this.inputDriver.getMousePos();
|
|
1029
898
|
console.log(`[HealthCheck] Current mouse position: (${startPos.x}, ${startPos.y})`);
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
} finally{
|
|
1045
|
-
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
1046
|
-
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
1047
|
-
}
|
|
899
|
+
const offsetX = Math.floor(40 * Math.random()) + 10;
|
|
900
|
+
const offsetY = Math.floor(40 * Math.random()) + 10;
|
|
901
|
+
const targetX = startPos.x + offsetX;
|
|
902
|
+
const targetY = startPos.y + offsetY;
|
|
903
|
+
console.log(`[HealthCheck] Moving mouse to (${targetX}, ${targetY})...`);
|
|
904
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
905
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
906
|
+
const movedPos = this.inputDriver.getMousePos();
|
|
907
|
+
console.log(`[HealthCheck] Mouse position after move: (${movedPos.x}, ${movedPos.y})`);
|
|
908
|
+
const deltaX = Math.abs(movedPos.x - targetX);
|
|
909
|
+
const deltaY = Math.abs(movedPos.y - targetY);
|
|
910
|
+
if (deltaX > 5 || deltaY > 5) {
|
|
911
|
+
const msg = `[HealthCheck] WARNING: Mouse control may not be working. Expected (${targetX}, ${targetY}), got (${movedPos.x}, ${movedPos.y}), delta=(${deltaX}, ${deltaY})`;
|
|
912
|
+
warnDevice(msg);
|
|
1048
913
|
}
|
|
1049
914
|
if ('win32' === process.platform && !this.isRunningAsAdmin()) {
|
|
1050
915
|
const hint = [
|
|
@@ -1054,7 +919,10 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
1054
919
|
].join(' ');
|
|
1055
920
|
warnDevice(`[HealthCheck] ${hint}`);
|
|
1056
921
|
}
|
|
922
|
+
this.inputDriver.moveMouse(startPos.x, startPos.y);
|
|
923
|
+
console.log(`[HealthCheck] Mouse restored to (${startPos.x}, ${startPos.y})`);
|
|
1057
924
|
console.log('[HealthCheck] Listing monitors...');
|
|
925
|
+
const displays = await ComputerDevice.listDisplays();
|
|
1058
926
|
if (displays.length > 0) {
|
|
1059
927
|
console.log(`[HealthCheck] Found ${displays.length} monitor(s):`);
|
|
1060
928
|
for (const display of displays){
|
|
@@ -1237,14 +1105,10 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1237
1105
|
resolveUntargetedScrollPoint(screenSize) {
|
|
1238
1106
|
if ('win32' === process.platform) {
|
|
1239
1107
|
const activeWindowRect = this.inputDriver.getActiveWindowRect();
|
|
1240
|
-
if (activeWindowRect) {
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
};
|
|
1245
|
-
const bounds = this.displayGeometry?.bounds;
|
|
1246
|
-
if (!bounds || activeWindowCenter.x >= bounds.x && activeWindowCenter.x < bounds.x + bounds.width && activeWindowCenter.y >= bounds.y && activeWindowCenter.y < bounds.y + bounds.height) return activeWindowCenter;
|
|
1247
|
-
}
|
|
1108
|
+
if (activeWindowRect) return {
|
|
1109
|
+
x: activeWindowRect.x + activeWindowRect.width / 2,
|
|
1110
|
+
y: activeWindowRect.y + activeWindowRect.height / 2
|
|
1111
|
+
};
|
|
1248
1112
|
}
|
|
1249
1113
|
return this.toGlobalPoint({
|
|
1250
1114
|
x: screenSize.width / 2,
|
|
@@ -1255,16 +1119,17 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1255
1119
|
if (param.locate) {
|
|
1256
1120
|
const element = param.locate;
|
|
1257
1121
|
const [x, y] = element.center;
|
|
1258
|
-
|
|
1122
|
+
const point = this.toGlobalPoint({
|
|
1259
1123
|
x,
|
|
1260
1124
|
y
|
|
1261
|
-
}
|
|
1125
|
+
});
|
|
1126
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1262
1127
|
return;
|
|
1263
1128
|
}
|
|
1264
1129
|
const screenSize = await this.size();
|
|
1265
1130
|
if ('win32' === process.platform && this.inputDriver.focusActiveWindow()) await this.inputDriver.delay(CLICK_FOCUS_SETTLE_DELAY);
|
|
1266
1131
|
const point = this.resolveUntargetedScrollPoint(screenSize);
|
|
1267
|
-
|
|
1132
|
+
this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y));
|
|
1268
1133
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1269
1134
|
return screenSize;
|
|
1270
1135
|
}
|
|
@@ -1353,18 +1218,19 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1353
1218
|
if (this.destroyed) return;
|
|
1354
1219
|
this.destroyed = true;
|
|
1355
1220
|
this.inputDriver.destroy();
|
|
1221
|
+
const keepXvfbAliveUntilProcessExit = this.options?.keepXvfbAliveUntilProcessExit === true;
|
|
1356
1222
|
if (this.xvfbInstance) {
|
|
1357
|
-
this.xvfbInstance.stop();
|
|
1223
|
+
if (!keepXvfbAliveUntilProcessExit) this.xvfbInstance.stop();
|
|
1358
1224
|
this.xvfbInstance = void 0;
|
|
1359
1225
|
}
|
|
1360
|
-
if (this.xvfbCleanup) {
|
|
1226
|
+
if (this.xvfbCleanup && !keepXvfbAliveUntilProcessExit) {
|
|
1361
1227
|
process.removeListener('exit', this.xvfbCleanup);
|
|
1362
|
-
process.removeListener('SIGTERM', this.xvfbCleanup);
|
|
1363
1228
|
this.xvfbCleanup = void 0;
|
|
1364
1229
|
}
|
|
1365
|
-
if (this.
|
|
1366
|
-
process.removeListener('SIGINT', this.
|
|
1367
|
-
this.
|
|
1230
|
+
if (this.xvfbSignalCleanup) {
|
|
1231
|
+
process.removeListener('SIGINT', this.xvfbSignalCleanup);
|
|
1232
|
+
process.removeListener('SIGTERM', this.xvfbSignalCleanup);
|
|
1233
|
+
this.xvfbSignalCleanup = void 0;
|
|
1368
1234
|
}
|
|
1369
1235
|
debugDevice('Computer device destroyed');
|
|
1370
1236
|
}
|
|
@@ -1380,7 +1246,7 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1380
1246
|
device_define_property(this, "destroyed", false);
|
|
1381
1247
|
device_define_property(this, "xvfbInstance", void 0);
|
|
1382
1248
|
device_define_property(this, "xvfbCleanup", void 0);
|
|
1383
|
-
device_define_property(this, "
|
|
1249
|
+
device_define_property(this, "xvfbSignalCleanup", void 0);
|
|
1384
1250
|
device_define_property(this, "inputDriver", new ComputerInputDriver({
|
|
1385
1251
|
getLibnut: ()=>device_libnut,
|
|
1386
1252
|
useAppleScript: ()=>this.useAppleScript,
|
|
@@ -1412,16 +1278,14 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1412
1278
|
},
|
|
1413
1279
|
holdDuration,
|
|
1414
1280
|
displayId: this.displayId,
|
|
1415
|
-
displayGeometry: this.displayGeometry
|
|
1281
|
+
displayGeometry: this.displayGeometry ? {
|
|
1282
|
+
screenIndex: this.displayGeometry.screenIndex,
|
|
1283
|
+
cgDisplayId: this.displayGeometry.cgDisplayId,
|
|
1284
|
+
bounds: this.displayGeometry.bounds
|
|
1285
|
+
} : void 0
|
|
1416
1286
|
});
|
|
1417
1287
|
const frontmostBefore = 'darwin' === process.platform ? readDarwinFrontmostApplication() : void 0;
|
|
1418
|
-
await this.
|
|
1419
|
-
x: targetX,
|
|
1420
|
-
y: targetY
|
|
1421
|
-
}, 'Mouse did not reach the tap target', {
|
|
1422
|
-
smoothSteps: SMOOTH_MOVE_STEPS_TAP,
|
|
1423
|
-
smoothDelay: SMOOTH_MOVE_DELAY_TAP
|
|
1424
|
-
});
|
|
1288
|
+
await this.inputDriver.smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
1425
1289
|
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'primary');
|
|
1426
1290
|
if (frontmostBefore && 'darwin' === process.platform) {
|
|
1427
1291
|
await (0, utils_namespaceObject.sleep)(CLICK_FOCUS_SETTLE_DELAY);
|
|
@@ -1433,43 +1297,42 @@ $g.Dispose(); $bmp.Dispose(); $ms.Dispose()
|
|
|
1433
1297
|
focusChanged
|
|
1434
1298
|
});
|
|
1435
1299
|
if (focusChanged) {
|
|
1436
|
-
|
|
1437
|
-
x: targetX,
|
|
1438
|
-
y: targetY
|
|
1439
|
-
}, 'Mouse did not reach the focus follow-up target');
|
|
1300
|
+
this.inputDriver.moveMouse(targetX, targetY);
|
|
1440
1301
|
await pressMouseAtGlobalPoint(this.inputDriver, targetX, targetY, holdDuration, 'focus-follow-up');
|
|
1441
1302
|
}
|
|
1442
1303
|
}
|
|
1443
1304
|
},
|
|
1444
1305
|
doubleClick: async ({ x, y })=>{
|
|
1445
|
-
|
|
1306
|
+
const target = this.toGlobalPoint({
|
|
1446
1307
|
x,
|
|
1447
1308
|
y
|
|
1448
|
-
}
|
|
1309
|
+
});
|
|
1310
|
+
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1449
1311
|
this.inputDriver.mouseClick('left', true);
|
|
1450
1312
|
},
|
|
1451
1313
|
rightClick: async ({ x, y })=>{
|
|
1452
|
-
|
|
1314
|
+
const target = this.toGlobalPoint({
|
|
1453
1315
|
x,
|
|
1454
1316
|
y
|
|
1455
|
-
}
|
|
1317
|
+
});
|
|
1318
|
+
this.inputDriver.moveMouse(Math.round(target.x), Math.round(target.y));
|
|
1456
1319
|
this.inputDriver.mouseClick('right');
|
|
1457
1320
|
},
|
|
1458
1321
|
hover: async ({ x, y })=>{
|
|
1459
|
-
|
|
1322
|
+
const target = this.toGlobalPoint({
|
|
1460
1323
|
x,
|
|
1461
1324
|
y
|
|
1462
|
-
}, 'Mouse did not reach the hover target', {
|
|
1463
|
-
smoothSteps: SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1464
|
-
smoothDelay: SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
1465
1325
|
});
|
|
1326
|
+
await this.inputDriver.smoothMoveMouse(Math.round(target.x), Math.round(target.y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
1466
1327
|
await this.inputDriver.delay(MOUSE_MOVE_EFFECT_WAIT);
|
|
1467
1328
|
},
|
|
1468
1329
|
dragAndDrop: async (from, to)=>{
|
|
1469
|
-
|
|
1330
|
+
const globalFrom = this.toGlobalPoint(from);
|
|
1331
|
+
const globalTo = this.toGlobalPoint(to);
|
|
1332
|
+
this.inputDriver.moveMouse(Math.round(globalFrom.x), Math.round(globalFrom.y));
|
|
1470
1333
|
await this.inputDriver.withMouseButton('left', async ()=>{
|
|
1471
1334
|
await this.inputDriver.delay(100);
|
|
1472
|
-
|
|
1335
|
+
this.inputDriver.moveMouse(Math.round(globalTo.x), Math.round(globalTo.y));
|
|
1473
1336
|
await this.inputDriver.delay(100);
|
|
1474
1337
|
});
|
|
1475
1338
|
}
|
|
@@ -2244,7 +2107,8 @@ function createLocalComputerDevice(opts) {
|
|
|
2244
2107
|
keyboardTypeDelay: opts?.keyboardTypeDelay,
|
|
2245
2108
|
keyboardDriver: opts?.keyboardDriver,
|
|
2246
2109
|
headless: opts?.headless,
|
|
2247
|
-
xvfbResolution: opts?.xvfbResolution
|
|
2110
|
+
xvfbResolution: opts?.xvfbResolution,
|
|
2111
|
+
keepXvfbAliveUntilProcessExit: opts?.keepXvfbAliveUntilProcessExit
|
|
2248
2112
|
});
|
|
2249
2113
|
}
|
|
2250
2114
|
function createRDPComputerDevice(opts) {
|
|
@@ -2390,6 +2254,9 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2390
2254
|
...void 0 !== keyboardTypeDelay ? {
|
|
2391
2255
|
keyboardTypeDelay
|
|
2392
2256
|
} : {},
|
|
2257
|
+
...this.options.keepXvfbAliveUntilProcessExit ? {
|
|
2258
|
+
keepXvfbAliveUntilProcessExit: true
|
|
2259
|
+
} : {},
|
|
2393
2260
|
...(0, agent_behavior_init_args_namespaceObject.extractAgentBehaviorInitArgs)(opts) ?? {},
|
|
2394
2261
|
...reportOptions ?? {}
|
|
2395
2262
|
};
|
|
@@ -2455,20 +2322,20 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
2455
2322
|
}
|
|
2456
2323
|
];
|
|
2457
2324
|
}
|
|
2458
|
-
constructor(
|
|
2459
|
-
super(
|
|
2325
|
+
constructor(options = {}){
|
|
2326
|
+
super(), agent_tools_define_property(this, "options", void 0), agent_tools_define_property(this, "lastInitArgsSignature", void 0), agent_tools_define_property(this, "initArgSpec", void 0), this.options = options, this.initArgSpec = {
|
|
2460
2327
|
namespace: 'computer',
|
|
2461
2328
|
shape: computerInitArgShape,
|
|
2462
2329
|
cli: {
|
|
2463
2330
|
preferBareKeys: true
|
|
2464
2331
|
},
|
|
2465
2332
|
adapt: (extracted)=>adaptComputerInitArgs(extracted)
|
|
2466
|
-
}
|
|
2333
|
+
};
|
|
2467
2334
|
}
|
|
2468
2335
|
}
|
|
2469
2336
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
2470
2337
|
function version() {
|
|
2471
|
-
const currentVersion = "1.12.2
|
|
2338
|
+
const currentVersion = "1.12.2";
|
|
2472
2339
|
console.log(`@midscene/computer v${currentVersion}`);
|
|
2473
2340
|
return currentVersion;
|
|
2474
2341
|
}
|