@midscene/computer 1.8.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +898 -274
- package/dist/es/index.mjs +450 -395
- package/dist/es/mcp-server.mjs +898 -274
- package/dist/lib/cli.js +879 -256
- package/dist/lib/index.js +452 -394
- package/dist/lib/mcp-server.js +880 -257
- package/dist/types/index.d.ts +34 -5
- package/dist/types/mcp-server.d.ts +25 -5
- package/package.json +3 -3
package/dist/lib/cli.js
CHANGED
|
@@ -118,15 +118,6 @@ function _define_property(obj, key, value) {
|
|
|
118
118
|
else obj[key] = value;
|
|
119
119
|
return obj;
|
|
120
120
|
}
|
|
121
|
-
const computerInputParamSchema = core_namespaceObject.z.object({
|
|
122
|
-
value: core_namespaceObject.z.string().describe('The text to input'),
|
|
123
|
-
mode: core_namespaceObject.z["enum"]([
|
|
124
|
-
'replace',
|
|
125
|
-
'clear',
|
|
126
|
-
'append'
|
|
127
|
-
]).default('replace').optional().describe('Input mode: replace, clear, or append'),
|
|
128
|
-
locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
|
|
129
|
-
});
|
|
130
121
|
const SMOOTH_MOVE_STEPS_TAP = 8;
|
|
131
122
|
const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
132
123
|
const SMOOTH_MOVE_DELAY_TAP = 8;
|
|
@@ -437,7 +428,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
437
428
|
}
|
|
438
429
|
async healthCheck() {
|
|
439
430
|
console.log('[HealthCheck] Starting health check...');
|
|
440
|
-
console.log("[HealthCheck] @midscene/computer v1.8.
|
|
431
|
+
console.log("[HealthCheck] @midscene/computer v1.8.1");
|
|
441
432
|
console.log('[HealthCheck] Taking screenshot...');
|
|
442
433
|
const screenshotTimeout = 15000;
|
|
443
434
|
let timeoutId;
|
|
@@ -503,21 +494,38 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
503
494
|
debugDevice('Taking screenshot', {
|
|
504
495
|
displayId: this.displayId
|
|
505
496
|
});
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
497
|
+
const options = {
|
|
498
|
+
format: 'png'
|
|
499
|
+
};
|
|
500
|
+
if (void 0 !== this.displayId) if ('darwin' === process.platform) {
|
|
501
|
+
const screenIndex = Number(this.displayId);
|
|
502
|
+
if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
|
|
503
|
+
} else options.screen = this.displayId;
|
|
504
|
+
debugDevice('Screenshot options', options);
|
|
505
|
+
const MAX_ATTEMPTS = 3;
|
|
506
|
+
const RETRY_DELAY_MS = 300;
|
|
507
|
+
let lastRawMessage = '';
|
|
508
|
+
for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
|
|
515
509
|
const buffer = await external_screenshot_desktop_default()(options);
|
|
510
|
+
if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
|
|
516
511
|
return (0, img_namespaceObject.createImgBase64ByFormat)('png', buffer.toString('base64'));
|
|
517
512
|
} catch (error) {
|
|
518
|
-
|
|
519
|
-
|
|
513
|
+
lastRawMessage = error instanceof Error ? error.message : String(error);
|
|
514
|
+
const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
|
|
515
|
+
const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
|
|
516
|
+
debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
|
|
517
|
+
if (!willRetry) break;
|
|
518
|
+
await (0, utils_namespaceObject.sleep)(RETRY_DELAY_MS);
|
|
520
519
|
}
|
|
520
|
+
if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
|
|
521
|
+
|
|
522
|
+
Please follow these steps:
|
|
523
|
+
1. Open System Settings > Privacy & Security > Screen Recording
|
|
524
|
+
2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
|
|
525
|
+
3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
|
|
526
|
+
|
|
527
|
+
Original error: ${lastRawMessage}`);
|
|
528
|
+
throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
|
|
521
529
|
}
|
|
522
530
|
async size() {
|
|
523
531
|
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
@@ -563,228 +571,111 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
563
571
|
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
564
572
|
await this.typeViaClipboard(text);
|
|
565
573
|
}
|
|
574
|
+
async selectAllAndDelete() {
|
|
575
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
576
|
+
if (this.useAppleScript) {
|
|
577
|
+
sendKeyViaAppleScript('a', [
|
|
578
|
+
'command'
|
|
579
|
+
]);
|
|
580
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
581
|
+
sendKeyViaAppleScript('backspace', []);
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
585
|
+
libnut.keyTap('a', [
|
|
586
|
+
modifier
|
|
587
|
+
]);
|
|
588
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
589
|
+
libnut.keyTap('backspace');
|
|
590
|
+
}
|
|
591
|
+
async pressKeyboardShortcut(keyName) {
|
|
592
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
593
|
+
const keys = keyName.split('+');
|
|
594
|
+
const modifiers = keys.slice(0, -1).map(normalizeKeyName);
|
|
595
|
+
const key = normalizePrimaryKey(keys[keys.length - 1]);
|
|
596
|
+
debugDevice('KeyboardPress', {
|
|
597
|
+
original: keyName,
|
|
598
|
+
key,
|
|
599
|
+
modifiers,
|
|
600
|
+
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
601
|
+
});
|
|
602
|
+
if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
|
|
603
|
+
else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
|
|
604
|
+
else libnut.keyTap(key);
|
|
605
|
+
}
|
|
606
|
+
async performScroll(param) {
|
|
607
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
608
|
+
if (param.locate) {
|
|
609
|
+
const element = param.locate;
|
|
610
|
+
const [x, y] = element.center;
|
|
611
|
+
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
612
|
+
}
|
|
613
|
+
const scrollType = param?.scrollType;
|
|
614
|
+
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
615
|
+
if (edgeSpec) {
|
|
616
|
+
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
617
|
+
if (this.useAppleScript) {
|
|
618
|
+
sendKeyViaAppleScript(edgeSpec.key);
|
|
619
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
620
|
+
return;
|
|
621
|
+
}
|
|
622
|
+
const [dx, dy] = edgeSpec.libnut;
|
|
623
|
+
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
624
|
+
libnut.scrollMouse(dx, dy);
|
|
625
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
626
|
+
}
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
if ('singleAction' === scrollType || !scrollType) {
|
|
630
|
+
const distance = param?.distance || 500;
|
|
631
|
+
const direction = param?.direction || 'down';
|
|
632
|
+
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
633
|
+
if (isKnownDirection) {
|
|
634
|
+
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
635
|
+
if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
636
|
+
}
|
|
637
|
+
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
638
|
+
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
639
|
+
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
640
|
+
for(let i = 0; i < pages; i++){
|
|
641
|
+
sendKeyViaAppleScript(key);
|
|
642
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
643
|
+
}
|
|
644
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
645
|
+
return;
|
|
646
|
+
}
|
|
647
|
+
const ticks = Math.ceil(distance / 100);
|
|
648
|
+
const directionMap = {
|
|
649
|
+
up: [
|
|
650
|
+
0,
|
|
651
|
+
ticks
|
|
652
|
+
],
|
|
653
|
+
down: [
|
|
654
|
+
0,
|
|
655
|
+
-ticks
|
|
656
|
+
],
|
|
657
|
+
left: [
|
|
658
|
+
-ticks,
|
|
659
|
+
0
|
|
660
|
+
],
|
|
661
|
+
right: [
|
|
662
|
+
ticks,
|
|
663
|
+
0
|
|
664
|
+
]
|
|
665
|
+
};
|
|
666
|
+
const [dx, dy] = directionMap[direction] || [
|
|
667
|
+
0,
|
|
668
|
+
-ticks
|
|
669
|
+
];
|
|
670
|
+
libnut.scrollMouse(dx, dy);
|
|
671
|
+
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
672
|
+
return;
|
|
673
|
+
}
|
|
674
|
+
throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
|
|
675
|
+
}
|
|
566
676
|
actionSpace() {
|
|
567
677
|
const defaultActions = [
|
|
568
|
-
(0, device_namespaceObject.
|
|
569
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
570
|
-
const element = param.locate;
|
|
571
|
-
external_node_assert_default()(element, 'Element not found, cannot tap');
|
|
572
|
-
const [x, y] = element.center;
|
|
573
|
-
const targetX = Math.round(x);
|
|
574
|
-
const targetY = Math.round(y);
|
|
575
|
-
await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
576
|
-
libnut.mouseToggle('down', 'left');
|
|
577
|
-
await (0, utils_namespaceObject.sleep)(CLICK_HOLD_DURATION);
|
|
578
|
-
libnut.mouseToggle('up', 'left');
|
|
579
|
-
}),
|
|
580
|
-
(0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
|
|
581
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
582
|
-
const element = param.locate;
|
|
583
|
-
external_node_assert_default()(element, 'Element not found, cannot double click');
|
|
584
|
-
const [x, y] = element.center;
|
|
585
|
-
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
586
|
-
libnut.mouseClick('left', true);
|
|
587
|
-
}),
|
|
588
|
-
(0, device_namespaceObject.defineActionRightClick)(async (param)=>{
|
|
589
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
590
|
-
const element = param.locate;
|
|
591
|
-
external_node_assert_default()(element, 'Element not found, cannot right click');
|
|
592
|
-
const [x, y] = element.center;
|
|
593
|
-
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
594
|
-
libnut.mouseClick('right');
|
|
595
|
-
}),
|
|
596
|
-
(0, device_namespaceObject.defineAction)({
|
|
597
|
-
name: 'MouseMove',
|
|
598
|
-
description: 'Move the mouse to the element',
|
|
599
|
-
interfaceAlias: 'aiHover',
|
|
600
|
-
paramSchema: device_namespaceObject.actionHoverParamSchema,
|
|
601
|
-
sample: {
|
|
602
|
-
locate: {
|
|
603
|
-
prompt: 'the navigation menu item "Products"'
|
|
604
|
-
}
|
|
605
|
-
},
|
|
606
|
-
call: async (param)=>{
|
|
607
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
608
|
-
const element = param.locate;
|
|
609
|
-
external_node_assert_default()(element, 'Element not found, cannot move mouse');
|
|
610
|
-
const [x, y] = element.center;
|
|
611
|
-
const targetX = Math.round(x);
|
|
612
|
-
const targetY = Math.round(y);
|
|
613
|
-
await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
614
|
-
await (0, utils_namespaceObject.sleep)(MOUSE_MOVE_EFFECT_WAIT);
|
|
615
|
-
}
|
|
616
|
-
}),
|
|
617
|
-
(0, device_namespaceObject.defineAction)({
|
|
618
|
-
name: 'Input',
|
|
619
|
-
description: 'Input text into the input field',
|
|
620
|
-
interfaceAlias: 'aiInput',
|
|
621
|
-
paramSchema: computerInputParamSchema,
|
|
622
|
-
sample: {
|
|
623
|
-
value: 'test@example.com',
|
|
624
|
-
locate: {
|
|
625
|
-
prompt: 'the email input field'
|
|
626
|
-
}
|
|
627
|
-
},
|
|
628
|
-
call: async (param)=>{
|
|
629
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
630
|
-
const element = param.locate;
|
|
631
|
-
if (element) {
|
|
632
|
-
const [x, y] = element.center;
|
|
633
|
-
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
634
|
-
libnut.mouseClick('left');
|
|
635
|
-
await (0, utils_namespaceObject.sleep)(INPUT_FOCUS_DELAY);
|
|
636
|
-
if ('append' !== param.mode) {
|
|
637
|
-
if (this.useAppleScript) {
|
|
638
|
-
sendKeyViaAppleScript('a', [
|
|
639
|
-
'command'
|
|
640
|
-
]);
|
|
641
|
-
await (0, utils_namespaceObject.sleep)(50);
|
|
642
|
-
sendKeyViaAppleScript('backspace', []);
|
|
643
|
-
} else {
|
|
644
|
-
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
645
|
-
libnut.keyTap('a', [
|
|
646
|
-
modifier
|
|
647
|
-
]);
|
|
648
|
-
await (0, utils_namespaceObject.sleep)(50);
|
|
649
|
-
libnut.keyTap('backspace');
|
|
650
|
-
}
|
|
651
|
-
await (0, utils_namespaceObject.sleep)(INPUT_CLEAR_DELAY);
|
|
652
|
-
}
|
|
653
|
-
}
|
|
654
|
-
if ('clear' === param.mode) return;
|
|
655
|
-
if (!param.value) return;
|
|
656
|
-
await this.smartTypeString(param.value);
|
|
657
|
-
}
|
|
658
|
-
}),
|
|
659
|
-
(0, device_namespaceObject.defineActionScroll)(async (param)=>{
|
|
660
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
661
|
-
if (param.locate) {
|
|
662
|
-
const element = param.locate;
|
|
663
|
-
const [x, y] = element.center;
|
|
664
|
-
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
665
|
-
}
|
|
666
|
-
const scrollType = param?.scrollType;
|
|
667
|
-
const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
|
|
668
|
-
if (edgeSpec) {
|
|
669
|
-
if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
670
|
-
if (this.useAppleScript) {
|
|
671
|
-
sendKeyViaAppleScript(edgeSpec.key);
|
|
672
|
-
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
673
|
-
return;
|
|
674
|
-
}
|
|
675
|
-
const [dx, dy] = edgeSpec.libnut;
|
|
676
|
-
for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
|
|
677
|
-
libnut.scrollMouse(dx, dy);
|
|
678
|
-
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
679
|
-
}
|
|
680
|
-
return;
|
|
681
|
-
}
|
|
682
|
-
if ('singleAction' === scrollType || !scrollType) {
|
|
683
|
-
const distance = param?.distance || 500;
|
|
684
|
-
const direction = param?.direction || 'down';
|
|
685
|
-
const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
|
|
686
|
-
if (isKnownDirection) {
|
|
687
|
-
const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
|
|
688
|
-
if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
689
|
-
}
|
|
690
|
-
if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
|
|
691
|
-
const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
|
|
692
|
-
const key = 'up' === direction ? 'pageup' : 'pagedown';
|
|
693
|
-
for(let i = 0; i < pages; i++){
|
|
694
|
-
sendKeyViaAppleScript(key);
|
|
695
|
-
await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
|
|
696
|
-
}
|
|
697
|
-
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
698
|
-
return;
|
|
699
|
-
}
|
|
700
|
-
const ticks = Math.ceil(distance / 100);
|
|
701
|
-
const directionMap = {
|
|
702
|
-
up: [
|
|
703
|
-
0,
|
|
704
|
-
ticks
|
|
705
|
-
],
|
|
706
|
-
down: [
|
|
707
|
-
0,
|
|
708
|
-
-ticks
|
|
709
|
-
],
|
|
710
|
-
left: [
|
|
711
|
-
-ticks,
|
|
712
|
-
0
|
|
713
|
-
],
|
|
714
|
-
right: [
|
|
715
|
-
ticks,
|
|
716
|
-
0
|
|
717
|
-
]
|
|
718
|
-
};
|
|
719
|
-
const [dx, dy] = directionMap[direction] || [
|
|
720
|
-
0,
|
|
721
|
-
-ticks
|
|
722
|
-
];
|
|
723
|
-
libnut.scrollMouse(dx, dy);
|
|
724
|
-
await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
|
|
725
|
-
return;
|
|
726
|
-
}
|
|
727
|
-
throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
|
|
728
|
-
}),
|
|
729
|
-
(0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
|
|
730
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
731
|
-
if (param.locate) {
|
|
732
|
-
const [x, y] = param.locate.center;
|
|
733
|
-
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
734
|
-
libnut.mouseClick('left');
|
|
735
|
-
await (0, utils_namespaceObject.sleep)(50);
|
|
736
|
-
}
|
|
737
|
-
const keys = param.keyName.split('+');
|
|
738
|
-
const modifiers = keys.slice(0, -1).map(normalizeKeyName);
|
|
739
|
-
const key = normalizePrimaryKey(keys[keys.length - 1]);
|
|
740
|
-
debugDevice('KeyboardPress', {
|
|
741
|
-
original: param.keyName,
|
|
742
|
-
key,
|
|
743
|
-
modifiers,
|
|
744
|
-
driver: this.useAppleScript ? "applescript" : 'libnut'
|
|
745
|
-
});
|
|
746
|
-
if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
|
|
747
|
-
else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
|
|
748
|
-
else libnut.keyTap(key);
|
|
749
|
-
}),
|
|
750
|
-
(0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
|
|
751
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
752
|
-
const from = param.from;
|
|
753
|
-
const to = param.to;
|
|
754
|
-
external_node_assert_default()(from, 'missing "from" param for drag and drop');
|
|
755
|
-
external_node_assert_default()(to, 'missing "to" param for drag and drop');
|
|
756
|
-
const [fromX, fromY] = from.center;
|
|
757
|
-
const [toX, toY] = to.center;
|
|
758
|
-
libnut.moveMouse(Math.round(fromX), Math.round(fromY));
|
|
759
|
-
libnut.mouseToggle('down', 'left');
|
|
760
|
-
await (0, utils_namespaceObject.sleep)(100);
|
|
761
|
-
libnut.moveMouse(Math.round(toX), Math.round(toY));
|
|
762
|
-
await (0, utils_namespaceObject.sleep)(100);
|
|
763
|
-
libnut.mouseToggle('up', 'left');
|
|
764
|
-
}),
|
|
765
|
-
(0, device_namespaceObject.defineActionClearInput)(async (param)=>{
|
|
766
|
-
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
767
|
-
const element = param.locate;
|
|
768
|
-
external_node_assert_default()(element, 'Element not found, cannot clear input');
|
|
769
|
-
const [x, y] = element.center;
|
|
770
|
-
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
771
|
-
libnut.mouseClick('left');
|
|
772
|
-
await (0, utils_namespaceObject.sleep)(100);
|
|
773
|
-
if (this.useAppleScript) {
|
|
774
|
-
sendKeyViaAppleScript('a', [
|
|
775
|
-
'command'
|
|
776
|
-
]);
|
|
777
|
-
await (0, utils_namespaceObject.sleep)(50);
|
|
778
|
-
sendKeyViaAppleScript('backspace', []);
|
|
779
|
-
} else {
|
|
780
|
-
const modifier = 'darwin' === process.platform ? 'command' : 'control';
|
|
781
|
-
libnut.keyTap('a', [
|
|
782
|
-
modifier
|
|
783
|
-
]);
|
|
784
|
-
libnut.keyTap('backspace');
|
|
785
|
-
}
|
|
786
|
-
await (0, utils_namespaceObject.sleep)(50);
|
|
787
|
-
})
|
|
678
|
+
...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives)
|
|
788
679
|
];
|
|
789
680
|
const platformActions = Object.values(createPlatformActions());
|
|
790
681
|
const customActions = this.options?.customActions || [];
|
|
@@ -822,6 +713,88 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
|
|
|
822
713
|
_define_property(this, "xvfbCleanup", void 0);
|
|
823
714
|
_define_property(this, "useAppleScript", void 0);
|
|
824
715
|
_define_property(this, "uri", void 0);
|
|
716
|
+
_define_property(this, "inputPrimitives", {
|
|
717
|
+
pointer: {
|
|
718
|
+
tap: async ({ x, y })=>{
|
|
719
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
720
|
+
const targetX = Math.round(x);
|
|
721
|
+
const targetY = Math.round(y);
|
|
722
|
+
await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
|
|
723
|
+
libnut.mouseToggle('down', 'left');
|
|
724
|
+
await (0, utils_namespaceObject.sleep)(CLICK_HOLD_DURATION);
|
|
725
|
+
libnut.mouseToggle('up', 'left');
|
|
726
|
+
},
|
|
727
|
+
doubleClick: async ({ x, y })=>{
|
|
728
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
729
|
+
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
730
|
+
libnut.mouseClick('left', true);
|
|
731
|
+
},
|
|
732
|
+
rightClick: async ({ x, y })=>{
|
|
733
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
734
|
+
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
735
|
+
libnut.mouseClick('right');
|
|
736
|
+
},
|
|
737
|
+
hover: async ({ x, y })=>{
|
|
738
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
739
|
+
await smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
|
|
740
|
+
await (0, utils_namespaceObject.sleep)(MOUSE_MOVE_EFFECT_WAIT);
|
|
741
|
+
},
|
|
742
|
+
dragAndDrop: async (from, to)=>{
|
|
743
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
744
|
+
libnut.moveMouse(Math.round(from.x), Math.round(from.y));
|
|
745
|
+
libnut.mouseToggle('down', 'left');
|
|
746
|
+
await (0, utils_namespaceObject.sleep)(100);
|
|
747
|
+
libnut.moveMouse(Math.round(to.x), Math.round(to.y));
|
|
748
|
+
await (0, utils_namespaceObject.sleep)(100);
|
|
749
|
+
libnut.mouseToggle('up', 'left');
|
|
750
|
+
}
|
|
751
|
+
},
|
|
752
|
+
keyboard: {
|
|
753
|
+
typeText: async (value, opts)=>{
|
|
754
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
755
|
+
const element = opts?.target;
|
|
756
|
+
if (element) {
|
|
757
|
+
const [x, y] = element.center;
|
|
758
|
+
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
759
|
+
libnut.mouseClick('left');
|
|
760
|
+
await (0, utils_namespaceObject.sleep)(INPUT_FOCUS_DELAY);
|
|
761
|
+
if (opts?.replace !== false) {
|
|
762
|
+
await this.selectAllAndDelete();
|
|
763
|
+
await (0, utils_namespaceObject.sleep)(INPUT_CLEAR_DELAY);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
await this.smartTypeString(value);
|
|
767
|
+
},
|
|
768
|
+
keyboardPress: async (keyName, opts)=>{
|
|
769
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
770
|
+
const target = opts?.target;
|
|
771
|
+
if (target) {
|
|
772
|
+
const [x, y] = target.center;
|
|
773
|
+
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
774
|
+
libnut.mouseClick('left');
|
|
775
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
776
|
+
}
|
|
777
|
+
await this.pressKeyboardShortcut(keyName);
|
|
778
|
+
},
|
|
779
|
+
clearInput: async (target)=>{
|
|
780
|
+
external_node_assert_default()(libnut, 'libnut not initialized');
|
|
781
|
+
if (target) {
|
|
782
|
+
const element = target;
|
|
783
|
+
const [x, y] = element.center;
|
|
784
|
+
libnut.moveMouse(Math.round(x), Math.round(y));
|
|
785
|
+
libnut.mouseClick('left');
|
|
786
|
+
await (0, utils_namespaceObject.sleep)(100);
|
|
787
|
+
}
|
|
788
|
+
await this.selectAllAndDelete();
|
|
789
|
+
await (0, utils_namespaceObject.sleep)(50);
|
|
790
|
+
}
|
|
791
|
+
},
|
|
792
|
+
scroll: {
|
|
793
|
+
scroll: async (param)=>{
|
|
794
|
+
await this.performScroll(param);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
});
|
|
825
798
|
this.options = options;
|
|
826
799
|
this.displayId = options?.displayId;
|
|
827
800
|
this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
|
|
@@ -836,10 +809,578 @@ function createPlatformActions() {
|
|
|
836
809
|
})
|
|
837
810
|
};
|
|
838
811
|
}
|
|
839
|
-
require("node:events");
|
|
840
|
-
require("node:readline");
|
|
841
|
-
|
|
842
|
-
|
|
812
|
+
const external_node_events_namespaceObject = require("node:events");
|
|
813
|
+
const external_node_readline_namespaceObject = require("node:readline");
|
|
814
|
+
const platformBinaryMap = {
|
|
815
|
+
darwin: {
|
|
816
|
+
directory: 'darwin',
|
|
817
|
+
fileName: 'rdp-helper'
|
|
818
|
+
},
|
|
819
|
+
linux: {
|
|
820
|
+
directory: 'linux',
|
|
821
|
+
fileName: 'rdp-helper'
|
|
822
|
+
},
|
|
823
|
+
win32: {
|
|
824
|
+
directory: 'win32',
|
|
825
|
+
fileName: 'rdp-helper.exe'
|
|
826
|
+
}
|
|
827
|
+
};
|
|
828
|
+
function getPlatformBinary(platform) {
|
|
829
|
+
if (platform in platformBinaryMap) return platformBinaryMap[platform];
|
|
830
|
+
}
|
|
831
|
+
function currentDirname() {
|
|
832
|
+
return __dirname;
|
|
833
|
+
}
|
|
834
|
+
function getRdpHelperBinaryPath() {
|
|
835
|
+
const platformBinary = getPlatformBinary(process.platform);
|
|
836
|
+
if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
|
|
837
|
+
const hereDir = currentDirname();
|
|
838
|
+
const candidateRoots = [
|
|
839
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '../..'),
|
|
840
|
+
(0, external_node_path_namespaceObject.resolve)(hereDir, '../../..')
|
|
841
|
+
];
|
|
842
|
+
for (const root of candidateRoots){
|
|
843
|
+
const binaryPath = (0, external_node_path_namespaceObject.resolve)(root, 'bin', platformBinary.directory, platformBinary.fileName);
|
|
844
|
+
if ((0, external_node_fs_namespaceObject.existsSync)(binaryPath)) return binaryPath;
|
|
845
|
+
}
|
|
846
|
+
throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
|
|
847
|
+
}
|
|
848
|
+
function backend_client_define_property(obj, key, value) {
|
|
849
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
850
|
+
value: value,
|
|
851
|
+
enumerable: true,
|
|
852
|
+
configurable: true,
|
|
853
|
+
writable: true
|
|
854
|
+
});
|
|
855
|
+
else obj[key] = value;
|
|
856
|
+
return obj;
|
|
857
|
+
}
|
|
858
|
+
const debug = (0, logger_namespaceObject.getDebug)('rdp:backend');
|
|
859
|
+
const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
|
|
860
|
+
const MAX_STDERR_LINES = 40;
|
|
861
|
+
class HelperProcessRDPBackendClient {
|
|
862
|
+
async connect(config) {
|
|
863
|
+
this.fatalHelperError = void 0;
|
|
864
|
+
await this.ensureHelperStarted();
|
|
865
|
+
const response = await this.send({
|
|
866
|
+
type: 'connect',
|
|
867
|
+
config
|
|
868
|
+
});
|
|
869
|
+
if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
|
|
870
|
+
this.connected = true;
|
|
871
|
+
this.fatalHelperError = void 0;
|
|
872
|
+
return response.info;
|
|
873
|
+
}
|
|
874
|
+
async disconnect() {
|
|
875
|
+
const child = this.child;
|
|
876
|
+
if (!child) return;
|
|
877
|
+
let disconnectError;
|
|
878
|
+
if (this.connected && null === child.exitCode) try {
|
|
879
|
+
const response = await this.send({
|
|
880
|
+
type: 'disconnect'
|
|
881
|
+
});
|
|
882
|
+
this.expectOk(response, 'disconnect');
|
|
883
|
+
} catch (error) {
|
|
884
|
+
disconnectError = error instanceof Error ? error : new Error(String(error));
|
|
885
|
+
}
|
|
886
|
+
this.connected = false;
|
|
887
|
+
this.fatalHelperError = void 0;
|
|
888
|
+
await this.shutdownHelper();
|
|
889
|
+
if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
|
|
890
|
+
}
|
|
891
|
+
async screenshotBase64() {
|
|
892
|
+
const response = await this.send({
|
|
893
|
+
type: 'screenshot'
|
|
894
|
+
});
|
|
895
|
+
if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
|
|
896
|
+
return response.base64;
|
|
897
|
+
}
|
|
898
|
+
async size() {
|
|
899
|
+
const response = await this.send({
|
|
900
|
+
type: 'size'
|
|
901
|
+
});
|
|
902
|
+
if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
|
|
903
|
+
return response.size;
|
|
904
|
+
}
|
|
905
|
+
async mouseMove(x, y) {
|
|
906
|
+
const response = await this.send({
|
|
907
|
+
type: 'mouseMove',
|
|
908
|
+
x,
|
|
909
|
+
y
|
|
910
|
+
});
|
|
911
|
+
this.expectOk(response, 'mouseMove');
|
|
912
|
+
}
|
|
913
|
+
async mouseButton(button, action) {
|
|
914
|
+
const response = await this.send({
|
|
915
|
+
type: 'mouseButton',
|
|
916
|
+
button,
|
|
917
|
+
action
|
|
918
|
+
});
|
|
919
|
+
this.expectOk(response, 'mouseButton');
|
|
920
|
+
}
|
|
921
|
+
async wheel(direction, amount, x, y) {
|
|
922
|
+
const response = await this.send({
|
|
923
|
+
type: 'wheel',
|
|
924
|
+
direction,
|
|
925
|
+
amount,
|
|
926
|
+
x,
|
|
927
|
+
y
|
|
928
|
+
});
|
|
929
|
+
this.expectOk(response, 'wheel');
|
|
930
|
+
}
|
|
931
|
+
async keyPress(keyName) {
|
|
932
|
+
const response = await this.send({
|
|
933
|
+
type: 'keyPress',
|
|
934
|
+
keyName
|
|
935
|
+
});
|
|
936
|
+
this.expectOk(response, 'keyPress');
|
|
937
|
+
}
|
|
938
|
+
async typeText(text) {
|
|
939
|
+
const response = await this.send({
|
|
940
|
+
type: 'typeText',
|
|
941
|
+
text
|
|
942
|
+
});
|
|
943
|
+
this.expectOk(response, 'typeText');
|
|
944
|
+
}
|
|
945
|
+
async clearInput() {
|
|
946
|
+
const response = await this.send({
|
|
947
|
+
type: 'clearInput'
|
|
948
|
+
});
|
|
949
|
+
this.expectOk(response, 'clearInput');
|
|
950
|
+
}
|
|
951
|
+
async ensureHelperStarted() {
|
|
952
|
+
if (this.child && null === this.child.exitCode) return;
|
|
953
|
+
const helperPath = this.resolveHelperPath();
|
|
954
|
+
debug('starting rdp helper', {
|
|
955
|
+
helperPath
|
|
956
|
+
});
|
|
957
|
+
const child = this.spawnFn(helperPath, [], {
|
|
958
|
+
stdio: [
|
|
959
|
+
'pipe',
|
|
960
|
+
'pipe',
|
|
961
|
+
'pipe'
|
|
962
|
+
]
|
|
963
|
+
});
|
|
964
|
+
child.stdout.setEncoding('utf8');
|
|
965
|
+
child.stderr.setEncoding('utf8');
|
|
966
|
+
this.child = child;
|
|
967
|
+
this.stderrLines.length = 0;
|
|
968
|
+
this.stdoutReader = (0, external_node_readline_namespaceObject.createInterface)({
|
|
969
|
+
input: child.stdout,
|
|
970
|
+
crlfDelay: 1 / 0
|
|
971
|
+
});
|
|
972
|
+
this.stderrReader = (0, external_node_readline_namespaceObject.createInterface)({
|
|
973
|
+
input: child.stderr,
|
|
974
|
+
crlfDelay: 1 / 0
|
|
975
|
+
});
|
|
976
|
+
this.stdoutReader.on('line', (line)=>{
|
|
977
|
+
this.handleStdoutLine(line);
|
|
978
|
+
});
|
|
979
|
+
this.stderrReader.on('line', (line)=>{
|
|
980
|
+
this.captureStderrLine(line);
|
|
981
|
+
});
|
|
982
|
+
child.on('exit', (code, signal)=>{
|
|
983
|
+
this.connected = false;
|
|
984
|
+
const error = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`);
|
|
985
|
+
this.fatalHelperError = error;
|
|
986
|
+
this.rejectPending(error);
|
|
987
|
+
this.disposeReaders();
|
|
988
|
+
this.child = void 0;
|
|
989
|
+
});
|
|
990
|
+
child.on('error', (error)=>{
|
|
991
|
+
this.connected = false;
|
|
992
|
+
const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`);
|
|
993
|
+
this.fatalHelperError = helperError;
|
|
994
|
+
this.rejectPending(helperError);
|
|
995
|
+
this.disposeReaders();
|
|
996
|
+
this.child = void 0;
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
handleStdoutLine(line) {
|
|
1000
|
+
if (!line.trim()) return;
|
|
1001
|
+
let parsed;
|
|
1002
|
+
try {
|
|
1003
|
+
parsed = JSON.parse(line);
|
|
1004
|
+
} catch (error) {
|
|
1005
|
+
const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`);
|
|
1006
|
+
this.rejectPending(protocolError);
|
|
1007
|
+
this.shutdownHelper(protocolError);
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
1010
|
+
const pending = this.pending.get(parsed.id);
|
|
1011
|
+
if (!pending) return void debug('dropping response for unknown request id', parsed);
|
|
1012
|
+
this.pending.delete(parsed.id);
|
|
1013
|
+
if (parsed.ok) return void pending.resolve(parsed.payload);
|
|
1014
|
+
pending.reject(this.createHelperError(parsed.error.message, parsed.error.code));
|
|
1015
|
+
}
|
|
1016
|
+
captureStderrLine(line) {
|
|
1017
|
+
if (!line.trim()) return;
|
|
1018
|
+
this.stderrLines.push(line);
|
|
1019
|
+
if (this.stderrLines.length > MAX_STDERR_LINES) this.stderrLines.shift();
|
|
1020
|
+
}
|
|
1021
|
+
async send(payload) {
|
|
1022
|
+
if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
|
|
1023
|
+
await this.ensureHelperStarted();
|
|
1024
|
+
const child = this.child;
|
|
1025
|
+
if (!child || null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
|
|
1026
|
+
const id = `req-${++this.nextRequestId}`;
|
|
1027
|
+
const request = {
|
|
1028
|
+
id,
|
|
1029
|
+
payload
|
|
1030
|
+
};
|
|
1031
|
+
return new Promise((resolve, reject)=>{
|
|
1032
|
+
this.pending.set(id, {
|
|
1033
|
+
resolve,
|
|
1034
|
+
reject
|
|
1035
|
+
});
|
|
1036
|
+
child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
|
|
1037
|
+
if (!error) return;
|
|
1038
|
+
this.pending.delete(id);
|
|
1039
|
+
reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`));
|
|
1040
|
+
});
|
|
1041
|
+
});
|
|
1042
|
+
}
|
|
1043
|
+
expectOk(response, actionName) {
|
|
1044
|
+
if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
|
|
1045
|
+
}
|
|
1046
|
+
rejectPending(error) {
|
|
1047
|
+
for (const { reject } of this.pending.values())reject(error);
|
|
1048
|
+
this.pending.clear();
|
|
1049
|
+
}
|
|
1050
|
+
createHelperError(message, code) {
|
|
1051
|
+
const stderrSummary = this.stderrLines.join('\n').trim();
|
|
1052
|
+
const suffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
|
|
1053
|
+
const error = new Error(`${message}${suffix}`);
|
|
1054
|
+
if (code) error.name = code;
|
|
1055
|
+
return error;
|
|
1056
|
+
}
|
|
1057
|
+
disposeReaders() {
|
|
1058
|
+
this.stdoutReader?.close();
|
|
1059
|
+
this.stderrReader?.close();
|
|
1060
|
+
this.stdoutReader = void 0;
|
|
1061
|
+
this.stderrReader = void 0;
|
|
1062
|
+
}
|
|
1063
|
+
async shutdownHelper(rootError) {
|
|
1064
|
+
const child = this.child;
|
|
1065
|
+
this.child = void 0;
|
|
1066
|
+
this.disposeReaders();
|
|
1067
|
+
if (!child) return;
|
|
1068
|
+
this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
|
|
1069
|
+
if (null !== child.exitCode) return;
|
|
1070
|
+
child.stdin.end();
|
|
1071
|
+
const exited = Promise.race([
|
|
1072
|
+
(0, external_node_events_namespaceObject.once)(child, 'exit'),
|
|
1073
|
+
new Promise((resolve)=>{
|
|
1074
|
+
setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
|
|
1075
|
+
})
|
|
1076
|
+
]);
|
|
1077
|
+
const result = await exited;
|
|
1078
|
+
if ('timeout' !== result) return;
|
|
1079
|
+
child.kill('SIGTERM');
|
|
1080
|
+
const terminated = Promise.race([
|
|
1081
|
+
(0, external_node_events_namespaceObject.once)(child, 'exit'),
|
|
1082
|
+
new Promise((resolve)=>{
|
|
1083
|
+
setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
|
|
1084
|
+
})
|
|
1085
|
+
]);
|
|
1086
|
+
const terminateResult = await terminated;
|
|
1087
|
+
if ('timeout' !== terminateResult) return;
|
|
1088
|
+
child.kill('SIGKILL');
|
|
1089
|
+
await (0, external_node_events_namespaceObject.once)(child, 'exit');
|
|
1090
|
+
}
|
|
1091
|
+
constructor(options){
|
|
1092
|
+
backend_client_define_property(this, "spawnFn", void 0);
|
|
1093
|
+
backend_client_define_property(this, "resolveHelperPath", void 0);
|
|
1094
|
+
backend_client_define_property(this, "child", void 0);
|
|
1095
|
+
backend_client_define_property(this, "stdoutReader", void 0);
|
|
1096
|
+
backend_client_define_property(this, "stderrReader", void 0);
|
|
1097
|
+
backend_client_define_property(this, "pending", new Map());
|
|
1098
|
+
backend_client_define_property(this, "stderrLines", []);
|
|
1099
|
+
backend_client_define_property(this, "nextRequestId", 0);
|
|
1100
|
+
backend_client_define_property(this, "connected", false);
|
|
1101
|
+
backend_client_define_property(this, "fatalHelperError", void 0);
|
|
1102
|
+
this.spawnFn = options?.spawnFn || external_node_child_process_namespaceObject.spawn;
|
|
1103
|
+
const overridePath = options?.helperPath;
|
|
1104
|
+
this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
function createDefaultRDPBackendClient() {
|
|
1108
|
+
return new HelperProcessRDPBackendClient();
|
|
1109
|
+
}
|
|
1110
|
+
function device_define_property(obj, key, value) {
|
|
1111
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
1112
|
+
value: value,
|
|
1113
|
+
enumerable: true,
|
|
1114
|
+
configurable: true,
|
|
1115
|
+
writable: true
|
|
1116
|
+
});
|
|
1117
|
+
else obj[key] = value;
|
|
1118
|
+
return obj;
|
|
1119
|
+
}
|
|
1120
|
+
const device_debug = (0, logger_namespaceObject.getDebug)('rdp:device');
|
|
1121
|
+
const device_SMOOTH_MOVE_STEPS_TAP = 8;
|
|
1122
|
+
const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
|
|
1123
|
+
const SMOOTH_MOVE_STEPS_DRAG = 12;
|
|
1124
|
+
const device_SMOOTH_MOVE_DELAY_TAP = 8;
|
|
1125
|
+
const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
|
|
1126
|
+
const SMOOTH_MOVE_DELAY_DRAG = 10;
|
|
1127
|
+
const device_MOUSE_MOVE_EFFECT_WAIT = 300;
|
|
1128
|
+
const device_CLICK_HOLD_DURATION = 50;
|
|
1129
|
+
const DRAG_HOLD_DURATION = 100;
|
|
1130
|
+
const device_INPUT_FOCUS_DELAY = 300;
|
|
1131
|
+
const device_INPUT_CLEAR_DELAY = 150;
|
|
1132
|
+
const device_SCROLL_STEP_DELAY = 100;
|
|
1133
|
+
const device_SCROLL_COMPLETE_DELAY = 500;
|
|
1134
|
+
const DEFAULT_SCROLL_DISTANCE = 480;
|
|
1135
|
+
const device_EDGE_SCROLL_STEPS = 10;
|
|
1136
|
+
const DEFAULT_SCROLL_STEP_AMOUNT = 120;
|
|
1137
|
+
class RDPDevice {
|
|
1138
|
+
describe() {
|
|
1139
|
+
const port = this.options.port || 3389;
|
|
1140
|
+
const username = this.options.username ? ` as ${this.options.username}` : '';
|
|
1141
|
+
const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
|
|
1142
|
+
return `RDP Device ${this.options.host}:${port}${username}${session}`;
|
|
1143
|
+
}
|
|
1144
|
+
async connect() {
|
|
1145
|
+
this.throwIfDestroyed();
|
|
1146
|
+
device_debug('connecting to rdp backend', {
|
|
1147
|
+
host: this.options.host,
|
|
1148
|
+
port: this.options.port,
|
|
1149
|
+
username: this.options.username
|
|
1150
|
+
});
|
|
1151
|
+
this.connectionInfo = await this.backend.connect(this.options);
|
|
1152
|
+
this.cursorPosition = [
|
|
1153
|
+
Math.round(this.connectionInfo.size.width / 2),
|
|
1154
|
+
Math.round(this.connectionInfo.size.height / 2)
|
|
1155
|
+
];
|
|
1156
|
+
}
|
|
1157
|
+
async screenshotBase64() {
|
|
1158
|
+
this.assertConnected();
|
|
1159
|
+
return this.backend.screenshotBase64();
|
|
1160
|
+
}
|
|
1161
|
+
async size() {
|
|
1162
|
+
this.assertConnected();
|
|
1163
|
+
return this.backend.size();
|
|
1164
|
+
}
|
|
1165
|
+
async destroy() {
|
|
1166
|
+
if (this.destroyed) return;
|
|
1167
|
+
this.destroyed = true;
|
|
1168
|
+
this.connectionInfo = void 0;
|
|
1169
|
+
this.cursorPosition = void 0;
|
|
1170
|
+
await this.backend.disconnect();
|
|
1171
|
+
}
|
|
1172
|
+
actionSpace() {
|
|
1173
|
+
const defaultActions = [
|
|
1174
|
+
...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives),
|
|
1175
|
+
(0, device_namespaceObject.defineAction)({
|
|
1176
|
+
name: 'ListDisplays',
|
|
1177
|
+
description: 'List all available displays/monitors',
|
|
1178
|
+
call: async ()=>{
|
|
1179
|
+
this.assertConnected();
|
|
1180
|
+
const size = await this.size();
|
|
1181
|
+
return [
|
|
1182
|
+
{
|
|
1183
|
+
id: this.connectionInfo?.sessionId || this.options.host,
|
|
1184
|
+
name: `RDP ${this.connectionInfo?.server || this.options.host} (${size.width}x${size.height})`,
|
|
1185
|
+
primary: true
|
|
1186
|
+
}
|
|
1187
|
+
];
|
|
1188
|
+
}
|
|
1189
|
+
})
|
|
1190
|
+
];
|
|
1191
|
+
return [
|
|
1192
|
+
...defaultActions,
|
|
1193
|
+
...this.options.customActions || []
|
|
1194
|
+
];
|
|
1195
|
+
}
|
|
1196
|
+
assertConnected() {
|
|
1197
|
+
this.throwIfDestroyed();
|
|
1198
|
+
if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
|
|
1199
|
+
}
|
|
1200
|
+
throwIfDestroyed() {
|
|
1201
|
+
if (this.destroyed) throw new Error('RDPDevice has been destroyed');
|
|
1202
|
+
}
|
|
1203
|
+
async moveToElement(element, options) {
|
|
1204
|
+
this.assertConnected();
|
|
1205
|
+
const targetX = Math.round(element.center[0]);
|
|
1206
|
+
const targetY = Math.round(element.center[1]);
|
|
1207
|
+
await this.movePointer(targetX, targetY, options);
|
|
1208
|
+
}
|
|
1209
|
+
async clearInput() {
|
|
1210
|
+
if (this.backend.clearInput) return void await this.backend.clearInput();
|
|
1211
|
+
await this.backend.keyPress('Control+A');
|
|
1212
|
+
await this.backend.keyPress('Backspace');
|
|
1213
|
+
}
|
|
1214
|
+
edgeScrollDirection(scrollType) {
|
|
1215
|
+
switch(scrollType){
|
|
1216
|
+
case 'scrollToTop':
|
|
1217
|
+
return 'up';
|
|
1218
|
+
case 'scrollToBottom':
|
|
1219
|
+
return 'down';
|
|
1220
|
+
case 'scrollToLeft':
|
|
1221
|
+
return 'left';
|
|
1222
|
+
case 'scrollToRight':
|
|
1223
|
+
return 'right';
|
|
1224
|
+
case 'singleAction':
|
|
1225
|
+
return 'down';
|
|
1226
|
+
default:
|
|
1227
|
+
throw new Error(`Unsupported scroll type: ${scrollType}`);
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
async movePointer(targetX, targetY, options) {
|
|
1231
|
+
this.assertConnected();
|
|
1232
|
+
const start = this.cursorPosition || [
|
|
1233
|
+
targetX,
|
|
1234
|
+
targetY
|
|
1235
|
+
];
|
|
1236
|
+
const steps = Math.max(1, options?.steps || 1);
|
|
1237
|
+
const stepDelayMs = options?.stepDelayMs || 0;
|
|
1238
|
+
for(let step = 1; step <= steps; step++){
|
|
1239
|
+
const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
|
|
1240
|
+
const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
|
|
1241
|
+
await this.backend.mouseMove(x, y);
|
|
1242
|
+
this.cursorPosition = [
|
|
1243
|
+
x,
|
|
1244
|
+
y
|
|
1245
|
+
];
|
|
1246
|
+
if (stepDelayMs > 0 && step < steps) await (0, utils_namespaceObject.sleep)(stepDelayMs);
|
|
1247
|
+
}
|
|
1248
|
+
if (options?.settleDelayMs) await (0, utils_namespaceObject.sleep)(options.settleDelayMs);
|
|
1249
|
+
}
|
|
1250
|
+
async performWheel(direction, amount, x, y) {
|
|
1251
|
+
let remaining = Math.abs(amount);
|
|
1252
|
+
if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
|
|
1253
|
+
while(remaining > 0){
|
|
1254
|
+
const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
|
|
1255
|
+
await this.backend.wheel(direction, chunk, x, y);
|
|
1256
|
+
remaining -= chunk;
|
|
1257
|
+
if (remaining > 0) await (0, utils_namespaceObject.sleep)(device_SCROLL_STEP_DELAY);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
constructor(options){
|
|
1261
|
+
device_define_property(this, "interfaceType", 'rdp');
|
|
1262
|
+
device_define_property(this, "options", void 0);
|
|
1263
|
+
device_define_property(this, "backend", void 0);
|
|
1264
|
+
device_define_property(this, "connectionInfo", void 0);
|
|
1265
|
+
device_define_property(this, "destroyed", false);
|
|
1266
|
+
device_define_property(this, "cursorPosition", void 0);
|
|
1267
|
+
device_define_property(this, "uri", void 0);
|
|
1268
|
+
device_define_property(this, "inputPrimitives", {
|
|
1269
|
+
pointer: {
|
|
1270
|
+
tap: async ({ x, y })=>{
|
|
1271
|
+
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1272
|
+
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1273
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1274
|
+
});
|
|
1275
|
+
await this.backend.mouseButton('left', 'down');
|
|
1276
|
+
await (0, utils_namespaceObject.sleep)(device_CLICK_HOLD_DURATION);
|
|
1277
|
+
await this.backend.mouseButton('left', 'up');
|
|
1278
|
+
},
|
|
1279
|
+
doubleClick: async ({ x, y })=>{
|
|
1280
|
+
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1281
|
+
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1282
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1283
|
+
});
|
|
1284
|
+
await this.backend.mouseButton('left', 'doubleClick');
|
|
1285
|
+
},
|
|
1286
|
+
rightClick: async ({ x, y })=>{
|
|
1287
|
+
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1288
|
+
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1289
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1290
|
+
});
|
|
1291
|
+
await this.backend.mouseButton('right', 'click');
|
|
1292
|
+
},
|
|
1293
|
+
hover: async ({ x, y })=>{
|
|
1294
|
+
await this.movePointer(Math.round(x), Math.round(y), {
|
|
1295
|
+
steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1296
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
|
|
1297
|
+
settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
|
|
1298
|
+
});
|
|
1299
|
+
},
|
|
1300
|
+
dragAndDrop: async (from, to)=>{
|
|
1301
|
+
await this.movePointer(Math.round(from.x), Math.round(from.y), {
|
|
1302
|
+
steps: device_SMOOTH_MOVE_STEPS_TAP,
|
|
1303
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
|
|
1304
|
+
});
|
|
1305
|
+
await this.backend.mouseButton('left', 'down');
|
|
1306
|
+
await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
|
|
1307
|
+
await this.movePointer(Math.round(to.x), Math.round(to.y), {
|
|
1308
|
+
steps: SMOOTH_MOVE_STEPS_DRAG,
|
|
1309
|
+
stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
|
|
1310
|
+
});
|
|
1311
|
+
await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
|
|
1312
|
+
await this.backend.mouseButton('left', 'up');
|
|
1313
|
+
}
|
|
1314
|
+
},
|
|
1315
|
+
keyboard: {
|
|
1316
|
+
typeText: async (value, opts)=>{
|
|
1317
|
+
this.assertConnected();
|
|
1318
|
+
const target = opts?.target;
|
|
1319
|
+
if (target) {
|
|
1320
|
+
await this.inputPrimitives.pointer.tap({
|
|
1321
|
+
x: target.center[0],
|
|
1322
|
+
y: target.center[1]
|
|
1323
|
+
});
|
|
1324
|
+
await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
|
|
1325
|
+
}
|
|
1326
|
+
if (opts?.replace !== false) {
|
|
1327
|
+
await this.clearInput();
|
|
1328
|
+
await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
|
|
1329
|
+
}
|
|
1330
|
+
if (opts?.focusOnly || !value) return;
|
|
1331
|
+
await this.backend.typeText(value);
|
|
1332
|
+
},
|
|
1333
|
+
clearInput: async (target)=>{
|
|
1334
|
+
this.assertConnected();
|
|
1335
|
+
const element = target;
|
|
1336
|
+
if (element) {
|
|
1337
|
+
await this.inputPrimitives.pointer.tap({
|
|
1338
|
+
x: element.center[0],
|
|
1339
|
+
y: element.center[1]
|
|
1340
|
+
});
|
|
1341
|
+
await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
|
|
1342
|
+
}
|
|
1343
|
+
await this.clearInput();
|
|
1344
|
+
await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
|
|
1345
|
+
},
|
|
1346
|
+
keyboardPress: async (keyName, opts)=>{
|
|
1347
|
+
this.assertConnected();
|
|
1348
|
+
const target = opts?.target;
|
|
1349
|
+
if (target) await this.inputPrimitives.pointer.tap({
|
|
1350
|
+
x: target.center[0],
|
|
1351
|
+
y: target.center[1]
|
|
1352
|
+
});
|
|
1353
|
+
await this.backend.keyPress(keyName);
|
|
1354
|
+
}
|
|
1355
|
+
},
|
|
1356
|
+
scroll: {
|
|
1357
|
+
scroll: async (param)=>{
|
|
1358
|
+
this.assertConnected();
|
|
1359
|
+
const target = param.locate;
|
|
1360
|
+
if (target) await this.moveToElement(target, {
|
|
1361
|
+
steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
|
|
1362
|
+
stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
|
|
1363
|
+
});
|
|
1364
|
+
if (param.scrollType && 'singleAction' !== param.scrollType) {
|
|
1365
|
+
const direction = this.edgeScrollDirection(param.scrollType);
|
|
1366
|
+
for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
|
|
1367
|
+
await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
|
|
1368
|
+
return;
|
|
1369
|
+
}
|
|
1370
|
+
await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
|
|
1371
|
+
await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
});
|
|
1375
|
+
this.options = {
|
|
1376
|
+
port: 3389,
|
|
1377
|
+
securityProtocol: 'auto',
|
|
1378
|
+
ignoreCertificate: false,
|
|
1379
|
+
...options
|
|
1380
|
+
};
|
|
1381
|
+
this.backend = options.backend || createDefaultRDPBackendClient();
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
843
1384
|
class ComputerAgent extends agent_namespaceObject.Agent {
|
|
844
1385
|
}
|
|
845
1386
|
function createLocalComputerDevice(opts) {
|
|
@@ -851,12 +1392,33 @@ function createLocalComputerDevice(opts) {
|
|
|
851
1392
|
xvfbResolution: opts?.xvfbResolution
|
|
852
1393
|
});
|
|
853
1394
|
}
|
|
1395
|
+
function createRDPComputerDevice(opts) {
|
|
1396
|
+
return new RDPDevice({
|
|
1397
|
+
host: opts.host,
|
|
1398
|
+
port: opts.port,
|
|
1399
|
+
username: opts.username,
|
|
1400
|
+
password: opts.password,
|
|
1401
|
+
domain: opts.domain,
|
|
1402
|
+
adminSession: opts.adminSession,
|
|
1403
|
+
ignoreCertificate: opts.ignoreCertificate,
|
|
1404
|
+
securityProtocol: opts.securityProtocol,
|
|
1405
|
+
desktopWidth: opts.desktopWidth,
|
|
1406
|
+
desktopHeight: opts.desktopHeight,
|
|
1407
|
+
backend: opts.backend,
|
|
1408
|
+
customActions: opts.customActions
|
|
1409
|
+
});
|
|
1410
|
+
}
|
|
854
1411
|
async function agentForComputer(opts) {
|
|
855
1412
|
const device = createLocalComputerDevice(opts);
|
|
856
1413
|
await device.connect();
|
|
857
1414
|
return new ComputerAgent(device, opts);
|
|
858
1415
|
}
|
|
859
1416
|
const agentFromComputer = agentForComputer;
|
|
1417
|
+
async function agentForRDPComputer(opts) {
|
|
1418
|
+
const device = createRDPComputerDevice(opts);
|
|
1419
|
+
await device.connect();
|
|
1420
|
+
return new ComputerAgent(device, opts);
|
|
1421
|
+
}
|
|
860
1422
|
function mcp_tools_define_property(obj, key, value) {
|
|
861
1423
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
862
1424
|
value: value,
|
|
@@ -868,10 +1430,61 @@ function mcp_tools_define_property(obj, key, value) {
|
|
|
868
1430
|
return obj;
|
|
869
1431
|
}
|
|
870
1432
|
const mcp_tools_debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
|
|
1433
|
+
const RDP_SECURITY_PROTOCOLS = [
|
|
1434
|
+
'auto',
|
|
1435
|
+
'tls',
|
|
1436
|
+
'nla',
|
|
1437
|
+
'rdp'
|
|
1438
|
+
];
|
|
871
1439
|
const computerInitArgShape = {
|
|
872
|
-
displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
|
|
873
|
-
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
|
|
1440
|
+
displayId: core_namespaceObject.z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
|
|
1441
|
+
headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
|
|
1442
|
+
host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
|
|
1443
|
+
port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
|
|
1444
|
+
username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
|
|
1445
|
+
password: core_namespaceObject.z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
|
|
1446
|
+
domain: core_namespaceObject.z.string().optional().describe('RDP domain. Requires host.'),
|
|
1447
|
+
adminSession: core_namespaceObject.z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
|
|
1448
|
+
ignoreCertificate: core_namespaceObject.z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
|
|
1449
|
+
securityProtocol: core_namespaceObject.z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
|
|
1450
|
+
desktopWidth: core_namespaceObject.z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
|
|
1451
|
+
desktopHeight: core_namespaceObject.z.number().optional().describe('Remote desktop height in pixels. Requires host.')
|
|
874
1452
|
};
|
|
1453
|
+
function adaptComputerInitArgs(extracted) {
|
|
1454
|
+
if (!extracted || 0 === Object.keys(extracted).length) return;
|
|
1455
|
+
if (extracted.host) {
|
|
1456
|
+
const { displayId: _d, headless: _h, ...rdpFields } = extracted;
|
|
1457
|
+
return {
|
|
1458
|
+
mode: 'rdp',
|
|
1459
|
+
...rdpFields,
|
|
1460
|
+
host: extracted.host
|
|
1461
|
+
};
|
|
1462
|
+
}
|
|
1463
|
+
return {
|
|
1464
|
+
mode: 'local',
|
|
1465
|
+
displayId: extracted.displayId,
|
|
1466
|
+
headless: extracted.headless
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
function shouldRetargetAgent(opts) {
|
|
1470
|
+
if (!opts) return false;
|
|
1471
|
+
if ('rdp' === opts.mode) return true;
|
|
1472
|
+
return void 0 !== opts.displayId || void 0 !== opts.headless;
|
|
1473
|
+
}
|
|
1474
|
+
function describeConnectTarget(opts) {
|
|
1475
|
+
if (opts?.mode === 'rdp') {
|
|
1476
|
+
const portSuffix = opts.port ? `:${opts.port}` : '';
|
|
1477
|
+
const userSuffix = opts.username ? ` as ${opts.username}` : '';
|
|
1478
|
+
return ` via RDP (${opts.host}${portSuffix}${userSuffix})`;
|
|
1479
|
+
}
|
|
1480
|
+
if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
|
|
1481
|
+
return ' (Primary display)';
|
|
1482
|
+
}
|
|
1483
|
+
function getCliReportSessionTarget(opts) {
|
|
1484
|
+
if (opts?.mode === 'rdp') return `rdp:${opts.host}`;
|
|
1485
|
+
if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
|
|
1486
|
+
return 'primary';
|
|
1487
|
+
}
|
|
875
1488
|
class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
|
|
876
1489
|
getCliReportSessionName() {
|
|
877
1490
|
return 'midscene-computer';
|
|
@@ -880,9 +1493,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
880
1493
|
return new ComputerDevice({});
|
|
881
1494
|
}
|
|
882
1495
|
async ensureAgent(opts) {
|
|
883
|
-
|
|
884
|
-
const headless = opts?.headless;
|
|
885
|
-
if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
|
|
1496
|
+
if (this.agent && shouldRetargetAgent(opts)) {
|
|
886
1497
|
try {
|
|
887
1498
|
await this.agent.destroy?.();
|
|
888
1499
|
} catch (error) {
|
|
@@ -891,8 +1502,20 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
891
1502
|
this.agent = void 0;
|
|
892
1503
|
}
|
|
893
1504
|
if (this.agent) return this.agent;
|
|
894
|
-
mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
895
1505
|
const reportOptions = this.readCliReportAgentOptions();
|
|
1506
|
+
if (opts?.mode === 'rdp') {
|
|
1507
|
+
mcp_tools_debug('Creating RDP Computer agent for host:', opts.host);
|
|
1508
|
+
const { mode: _mode, ...rdpFields } = opts;
|
|
1509
|
+
const agent = await agentForRDPComputer({
|
|
1510
|
+
...rdpFields,
|
|
1511
|
+
...reportOptions ?? {}
|
|
1512
|
+
});
|
|
1513
|
+
this.agent = agent;
|
|
1514
|
+
return agent;
|
|
1515
|
+
}
|
|
1516
|
+
const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
|
|
1517
|
+
const headless = opts?.mode === 'local' ? opts.headless : void 0;
|
|
1518
|
+
mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
|
|
896
1519
|
const agentOpts = {
|
|
897
1520
|
...displayId ? {
|
|
898
1521
|
displayId
|
|
@@ -910,12 +1533,12 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
910
1533
|
return [
|
|
911
1534
|
{
|
|
912
1535
|
name: 'computer_connect',
|
|
913
|
-
description:
|
|
1536
|
+
description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
|
|
914
1537
|
schema: this.getAgentInitArgSchema(),
|
|
915
1538
|
cli: this.getAgentInitArgCliMetadata(),
|
|
916
1539
|
handler: async (args)=>{
|
|
917
1540
|
const initArgs = this.extractAgentInitParam(args);
|
|
918
|
-
const reportSession = this.createNewCliReportSession(initArgs
|
|
1541
|
+
const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
|
|
919
1542
|
this.commitCliReportSession(reportSession);
|
|
920
1543
|
if (this.agent) {
|
|
921
1544
|
try {
|
|
@@ -931,7 +1554,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
931
1554
|
content: [
|
|
932
1555
|
{
|
|
933
1556
|
type: 'text',
|
|
934
|
-
text: `Connected to computer${
|
|
1557
|
+
text: `Connected to computer${describeConnectTarget(initArgs)}`
|
|
935
1558
|
},
|
|
936
1559
|
...this.buildScreenshotContent(screenshot)
|
|
937
1560
|
]
|
|
@@ -969,14 +1592,14 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
|
|
|
969
1592
|
cli: {
|
|
970
1593
|
preferBareKeys: true
|
|
971
1594
|
},
|
|
972
|
-
adapt: (extracted)=>extracted
|
|
1595
|
+
adapt: (extracted)=>adaptComputerInitArgs(extracted)
|
|
973
1596
|
});
|
|
974
1597
|
}
|
|
975
1598
|
}
|
|
976
1599
|
const tools = new ComputerMidsceneTools();
|
|
977
1600
|
(0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
|
|
978
1601
|
stripPrefix: 'computer_',
|
|
979
|
-
version: "1.8.
|
|
1602
|
+
version: "1.8.1",
|
|
980
1603
|
extraCommands: (0, core_namespaceObject.createReportCliCommands)()
|
|
981
1604
|
}).catch((e)=>{
|
|
982
1605
|
process.exit((0, cli_namespaceObject.reportCLIError)(e));
|