@midscene/computer 1.8.0 → 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/lib/cli.js CHANGED
@@ -118,15 +118,6 @@ function _define_property(obj, key, value) {
118
118
  else obj[key] = value;
119
119
  return obj;
120
120
  }
121
- const computerInputParamSchema = core_namespaceObject.z.object({
122
- value: core_namespaceObject.z.string().describe('The text to input'),
123
- mode: core_namespaceObject.z["enum"]([
124
- 'replace',
125
- 'clear',
126
- 'append'
127
- ]).default('replace').optional().describe('Input mode: replace, clear, or append'),
128
- locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
129
- });
130
121
  const SMOOTH_MOVE_STEPS_TAP = 8;
131
122
  const SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
132
123
  const SMOOTH_MOVE_DELAY_TAP = 8;
@@ -437,7 +428,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
437
428
  }
438
429
  async healthCheck() {
439
430
  console.log('[HealthCheck] Starting health check...');
440
- console.log("[HealthCheck] @midscene/computer v1.8.0");
431
+ console.log("[HealthCheck] @midscene/computer v1.8.1");
441
432
  console.log('[HealthCheck] Taking screenshot...');
442
433
  const screenshotTimeout = 15000;
443
434
  let timeoutId;
@@ -503,21 +494,38 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
503
494
  debugDevice('Taking screenshot', {
504
495
  displayId: this.displayId
505
496
  });
506
- try {
507
- const options = {
508
- format: 'png'
509
- };
510
- if (void 0 !== this.displayId) if ('darwin' === process.platform) {
511
- const screenIndex = Number(this.displayId);
512
- if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
513
- } else options.screen = this.displayId;
514
- debugDevice('Screenshot options', options);
497
+ const options = {
498
+ format: 'png'
499
+ };
500
+ if (void 0 !== this.displayId) if ('darwin' === process.platform) {
501
+ const screenIndex = Number(this.displayId);
502
+ if (!Number.isNaN(screenIndex)) options.screen = screenIndex;
503
+ } else options.screen = this.displayId;
504
+ debugDevice('Screenshot options', options);
505
+ const MAX_ATTEMPTS = 3;
506
+ const RETRY_DELAY_MS = 300;
507
+ let lastRawMessage = '';
508
+ for(let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)try {
515
509
  const buffer = await external_screenshot_desktop_default()(options);
510
+ if (attempt > 1) debugDevice(`Screenshot succeeded on attempt ${attempt}`);
516
511
  return (0, img_namespaceObject.createImgBase64ByFormat)('png', buffer.toString('base64'));
517
512
  } catch (error) {
518
- debugDevice(`Screenshot failed: ${error}`);
519
- throw new Error(`Failed to take screenshot: ${error}`);
513
+ lastRawMessage = error instanceof Error ? error.message : String(error);
514
+ const isMacTransient = 'darwin' === process.platform && /could not create image from display/i.test(lastRawMessage);
515
+ const willRetry = isMacTransient && attempt < MAX_ATTEMPTS;
516
+ debugDevice(`Screenshot attempt ${attempt} failed: ${lastRawMessage}${willRetry ? ' — retrying' : ''}`);
517
+ if (!willRetry) break;
518
+ await (0, utils_namespaceObject.sleep)(RETRY_DELAY_MS);
520
519
  }
520
+ if ('darwin' === process.platform && /could not create image from display/i.test(lastRawMessage)) throw new Error(`Failed to take screenshot on macOS: the host process is missing Screen Recording permission, or the target display is locked/sleeping.
521
+
522
+ Please follow these steps:
523
+ 1. Open System Settings > Privacy & Security > Screen Recording
524
+ 2. Enable the application running this script (e.g., Terminal, iTerm2, VS Code, WebStorm, or Midscene Studio)
525
+ 3. Fully quit and relaunch that application after granting permission — macOS only re-reads this permission on process launch.
526
+
527
+ Original error: ${lastRawMessage}`);
528
+ throw new Error(`Failed to take screenshot: ${lastRawMessage}`);
521
529
  }
522
530
  async size() {
523
531
  external_node_assert_default()(libnut, 'libnut not initialized');
@@ -563,228 +571,111 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
563
571
  external_node_assert_default()(libnut, 'libnut not initialized');
564
572
  await this.typeViaClipboard(text);
565
573
  }
574
+ async selectAllAndDelete() {
575
+ external_node_assert_default()(libnut, 'libnut not initialized');
576
+ if (this.useAppleScript) {
577
+ sendKeyViaAppleScript('a', [
578
+ 'command'
579
+ ]);
580
+ await (0, utils_namespaceObject.sleep)(50);
581
+ sendKeyViaAppleScript('backspace', []);
582
+ return;
583
+ }
584
+ const modifier = 'darwin' === process.platform ? 'command' : 'control';
585
+ libnut.keyTap('a', [
586
+ modifier
587
+ ]);
588
+ await (0, utils_namespaceObject.sleep)(50);
589
+ libnut.keyTap('backspace');
590
+ }
591
+ async pressKeyboardShortcut(keyName) {
592
+ external_node_assert_default()(libnut, 'libnut not initialized');
593
+ const keys = keyName.split('+');
594
+ const modifiers = keys.slice(0, -1).map(normalizeKeyName);
595
+ const key = normalizePrimaryKey(keys[keys.length - 1]);
596
+ debugDevice('KeyboardPress', {
597
+ original: keyName,
598
+ key,
599
+ modifiers,
600
+ driver: this.useAppleScript ? "applescript" : 'libnut'
601
+ });
602
+ if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
603
+ else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
604
+ else libnut.keyTap(key);
605
+ }
606
+ async performScroll(param) {
607
+ external_node_assert_default()(libnut, 'libnut not initialized');
608
+ if (param.locate) {
609
+ const element = param.locate;
610
+ const [x, y] = element.center;
611
+ libnut.moveMouse(Math.round(x), Math.round(y));
612
+ }
613
+ const scrollType = param?.scrollType;
614
+ const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
615
+ if (edgeSpec) {
616
+ if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
617
+ if (this.useAppleScript) {
618
+ sendKeyViaAppleScript(edgeSpec.key);
619
+ await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
620
+ return;
621
+ }
622
+ const [dx, dy] = edgeSpec.libnut;
623
+ for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
624
+ libnut.scrollMouse(dx, dy);
625
+ await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
626
+ }
627
+ return;
628
+ }
629
+ if ('singleAction' === scrollType || !scrollType) {
630
+ const distance = param?.distance || 500;
631
+ const direction = param?.direction || 'down';
632
+ const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
633
+ if (isKnownDirection) {
634
+ const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
635
+ if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
636
+ }
637
+ if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
638
+ const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
639
+ const key = 'up' === direction ? 'pageup' : 'pagedown';
640
+ for(let i = 0; i < pages; i++){
641
+ sendKeyViaAppleScript(key);
642
+ await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
643
+ }
644
+ await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
645
+ return;
646
+ }
647
+ const ticks = Math.ceil(distance / 100);
648
+ const directionMap = {
649
+ up: [
650
+ 0,
651
+ ticks
652
+ ],
653
+ down: [
654
+ 0,
655
+ -ticks
656
+ ],
657
+ left: [
658
+ -ticks,
659
+ 0
660
+ ],
661
+ right: [
662
+ ticks,
663
+ 0
664
+ ]
665
+ };
666
+ const [dx, dy] = directionMap[direction] || [
667
+ 0,
668
+ -ticks
669
+ ];
670
+ libnut.scrollMouse(dx, dy);
671
+ await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
672
+ return;
673
+ }
674
+ throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
675
+ }
566
676
  actionSpace() {
567
677
  const defaultActions = [
568
- (0, device_namespaceObject.defineActionTap)(async (param)=>{
569
- external_node_assert_default()(libnut, 'libnut not initialized');
570
- const element = param.locate;
571
- external_node_assert_default()(element, 'Element not found, cannot tap');
572
- const [x, y] = element.center;
573
- const targetX = Math.round(x);
574
- const targetY = Math.round(y);
575
- await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
576
- libnut.mouseToggle('down', 'left');
577
- await (0, utils_namespaceObject.sleep)(CLICK_HOLD_DURATION);
578
- libnut.mouseToggle('up', 'left');
579
- }),
580
- (0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
581
- external_node_assert_default()(libnut, 'libnut not initialized');
582
- const element = param.locate;
583
- external_node_assert_default()(element, 'Element not found, cannot double click');
584
- const [x, y] = element.center;
585
- libnut.moveMouse(Math.round(x), Math.round(y));
586
- libnut.mouseClick('left', true);
587
- }),
588
- (0, device_namespaceObject.defineActionRightClick)(async (param)=>{
589
- external_node_assert_default()(libnut, 'libnut not initialized');
590
- const element = param.locate;
591
- external_node_assert_default()(element, 'Element not found, cannot right click');
592
- const [x, y] = element.center;
593
- libnut.moveMouse(Math.round(x), Math.round(y));
594
- libnut.mouseClick('right');
595
- }),
596
- (0, device_namespaceObject.defineAction)({
597
- name: 'MouseMove',
598
- description: 'Move the mouse to the element',
599
- interfaceAlias: 'aiHover',
600
- paramSchema: device_namespaceObject.actionHoverParamSchema,
601
- sample: {
602
- locate: {
603
- prompt: 'the navigation menu item "Products"'
604
- }
605
- },
606
- call: async (param)=>{
607
- external_node_assert_default()(libnut, 'libnut not initialized');
608
- const element = param.locate;
609
- external_node_assert_default()(element, 'Element not found, cannot move mouse');
610
- const [x, y] = element.center;
611
- const targetX = Math.round(x);
612
- const targetY = Math.round(y);
613
- await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
614
- await (0, utils_namespaceObject.sleep)(MOUSE_MOVE_EFFECT_WAIT);
615
- }
616
- }),
617
- (0, device_namespaceObject.defineAction)({
618
- name: 'Input',
619
- description: 'Input text into the input field',
620
- interfaceAlias: 'aiInput',
621
- paramSchema: computerInputParamSchema,
622
- sample: {
623
- value: 'test@example.com',
624
- locate: {
625
- prompt: 'the email input field'
626
- }
627
- },
628
- call: async (param)=>{
629
- external_node_assert_default()(libnut, 'libnut not initialized');
630
- const element = param.locate;
631
- if (element) {
632
- const [x, y] = element.center;
633
- libnut.moveMouse(Math.round(x), Math.round(y));
634
- libnut.mouseClick('left');
635
- await (0, utils_namespaceObject.sleep)(INPUT_FOCUS_DELAY);
636
- if ('append' !== param.mode) {
637
- if (this.useAppleScript) {
638
- sendKeyViaAppleScript('a', [
639
- 'command'
640
- ]);
641
- await (0, utils_namespaceObject.sleep)(50);
642
- sendKeyViaAppleScript('backspace', []);
643
- } else {
644
- const modifier = 'darwin' === process.platform ? 'command' : 'control';
645
- libnut.keyTap('a', [
646
- modifier
647
- ]);
648
- await (0, utils_namespaceObject.sleep)(50);
649
- libnut.keyTap('backspace');
650
- }
651
- await (0, utils_namespaceObject.sleep)(INPUT_CLEAR_DELAY);
652
- }
653
- }
654
- if ('clear' === param.mode) return;
655
- if (!param.value) return;
656
- await this.smartTypeString(param.value);
657
- }
658
- }),
659
- (0, device_namespaceObject.defineActionScroll)(async (param)=>{
660
- external_node_assert_default()(libnut, 'libnut not initialized');
661
- if (param.locate) {
662
- const element = param.locate;
663
- const [x, y] = element.center;
664
- libnut.moveMouse(Math.round(x), Math.round(y));
665
- }
666
- const scrollType = param?.scrollType;
667
- const edgeSpec = scrollType && scrollType in EDGE_SCROLL_SPEC ? EDGE_SCROLL_SPEC[scrollType] : null;
668
- if (edgeSpec) {
669
- if (runPhasedScroll(edgeSpec.direction, EDGE_SCROLL_TOTAL_PX, EDGE_SCROLL_STEPS)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
670
- if (this.useAppleScript) {
671
- sendKeyViaAppleScript(edgeSpec.key);
672
- await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
673
- return;
674
- }
675
- const [dx, dy] = edgeSpec.libnut;
676
- for(let i = 0; i < SCROLL_REPEAT_COUNT; i++){
677
- libnut.scrollMouse(dx, dy);
678
- await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
679
- }
680
- return;
681
- }
682
- if ('singleAction' === scrollType || !scrollType) {
683
- const distance = param?.distance || 500;
684
- const direction = param?.direction || 'down';
685
- const isKnownDirection = 'up' === direction || 'down' === direction || 'left' === direction || 'right' === direction;
686
- if (isKnownDirection) {
687
- const steps = Math.max(PHASED_MIN_STEPS, Math.round(distance / PHASED_PIXELS_PER_STEP));
688
- if (runPhasedScroll(direction, distance, steps)) return void await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
689
- }
690
- if (this.useAppleScript && ('up' === direction || 'down' === direction)) {
691
- const pages = Math.max(1, Math.round(distance / APPROX_VIEWPORT_HEIGHT_PX));
692
- const key = 'up' === direction ? 'pageup' : 'pagedown';
693
- for(let i = 0; i < pages; i++){
694
- sendKeyViaAppleScript(key);
695
- await (0, utils_namespaceObject.sleep)(SCROLL_STEP_DELAY);
696
- }
697
- await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
698
- return;
699
- }
700
- const ticks = Math.ceil(distance / 100);
701
- const directionMap = {
702
- up: [
703
- 0,
704
- ticks
705
- ],
706
- down: [
707
- 0,
708
- -ticks
709
- ],
710
- left: [
711
- -ticks,
712
- 0
713
- ],
714
- right: [
715
- ticks,
716
- 0
717
- ]
718
- };
719
- const [dx, dy] = directionMap[direction] || [
720
- 0,
721
- -ticks
722
- ];
723
- libnut.scrollMouse(dx, dy);
724
- await (0, utils_namespaceObject.sleep)(SCROLL_COMPLETE_DELAY);
725
- return;
726
- }
727
- throw new Error(`Unknown scroll type: ${scrollType}, param: ${JSON.stringify(param)}`);
728
- }),
729
- (0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
730
- external_node_assert_default()(libnut, 'libnut not initialized');
731
- if (param.locate) {
732
- const [x, y] = param.locate.center;
733
- libnut.moveMouse(Math.round(x), Math.round(y));
734
- libnut.mouseClick('left');
735
- await (0, utils_namespaceObject.sleep)(50);
736
- }
737
- const keys = param.keyName.split('+');
738
- const modifiers = keys.slice(0, -1).map(normalizeKeyName);
739
- const key = normalizePrimaryKey(keys[keys.length - 1]);
740
- debugDevice('KeyboardPress', {
741
- original: param.keyName,
742
- key,
743
- modifiers,
744
- driver: this.useAppleScript ? "applescript" : 'libnut'
745
- });
746
- if (this.useAppleScript) sendKeyViaAppleScript(key, modifiers);
747
- else if (modifiers.length > 0) libnut.keyTap(key, modifiers);
748
- else libnut.keyTap(key);
749
- }),
750
- (0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
751
- external_node_assert_default()(libnut, 'libnut not initialized');
752
- const from = param.from;
753
- const to = param.to;
754
- external_node_assert_default()(from, 'missing "from" param for drag and drop');
755
- external_node_assert_default()(to, 'missing "to" param for drag and drop');
756
- const [fromX, fromY] = from.center;
757
- const [toX, toY] = to.center;
758
- libnut.moveMouse(Math.round(fromX), Math.round(fromY));
759
- libnut.mouseToggle('down', 'left');
760
- await (0, utils_namespaceObject.sleep)(100);
761
- libnut.moveMouse(Math.round(toX), Math.round(toY));
762
- await (0, utils_namespaceObject.sleep)(100);
763
- libnut.mouseToggle('up', 'left');
764
- }),
765
- (0, device_namespaceObject.defineActionClearInput)(async (param)=>{
766
- external_node_assert_default()(libnut, 'libnut not initialized');
767
- const element = param.locate;
768
- external_node_assert_default()(element, 'Element not found, cannot clear input');
769
- const [x, y] = element.center;
770
- libnut.moveMouse(Math.round(x), Math.round(y));
771
- libnut.mouseClick('left');
772
- await (0, utils_namespaceObject.sleep)(100);
773
- if (this.useAppleScript) {
774
- sendKeyViaAppleScript('a', [
775
- 'command'
776
- ]);
777
- await (0, utils_namespaceObject.sleep)(50);
778
- sendKeyViaAppleScript('backspace', []);
779
- } else {
780
- const modifier = 'darwin' === process.platform ? 'command' : 'control';
781
- libnut.keyTap('a', [
782
- modifier
783
- ]);
784
- libnut.keyTap('backspace');
785
- }
786
- await (0, utils_namespaceObject.sleep)(50);
787
- })
678
+ ...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives)
788
679
  ];
789
680
  const platformActions = Object.values(createPlatformActions());
790
681
  const customActions = this.options?.customActions || [];
@@ -822,6 +713,88 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
822
713
  _define_property(this, "xvfbCleanup", void 0);
823
714
  _define_property(this, "useAppleScript", void 0);
824
715
  _define_property(this, "uri", void 0);
716
+ _define_property(this, "inputPrimitives", {
717
+ pointer: {
718
+ tap: async ({ x, y })=>{
719
+ external_node_assert_default()(libnut, 'libnut not initialized');
720
+ const targetX = Math.round(x);
721
+ const targetY = Math.round(y);
722
+ await smoothMoveMouse(targetX, targetY, SMOOTH_MOVE_STEPS_TAP, SMOOTH_MOVE_DELAY_TAP);
723
+ libnut.mouseToggle('down', 'left');
724
+ await (0, utils_namespaceObject.sleep)(CLICK_HOLD_DURATION);
725
+ libnut.mouseToggle('up', 'left');
726
+ },
727
+ doubleClick: async ({ x, y })=>{
728
+ external_node_assert_default()(libnut, 'libnut not initialized');
729
+ libnut.moveMouse(Math.round(x), Math.round(y));
730
+ libnut.mouseClick('left', true);
731
+ },
732
+ rightClick: async ({ x, y })=>{
733
+ external_node_assert_default()(libnut, 'libnut not initialized');
734
+ libnut.moveMouse(Math.round(x), Math.round(y));
735
+ libnut.mouseClick('right');
736
+ },
737
+ hover: async ({ x, y })=>{
738
+ external_node_assert_default()(libnut, 'libnut not initialized');
739
+ await smoothMoveMouse(Math.round(x), Math.round(y), SMOOTH_MOVE_STEPS_MOUSE_MOVE, SMOOTH_MOVE_DELAY_MOUSE_MOVE);
740
+ await (0, utils_namespaceObject.sleep)(MOUSE_MOVE_EFFECT_WAIT);
741
+ },
742
+ dragAndDrop: async (from, to)=>{
743
+ external_node_assert_default()(libnut, 'libnut not initialized');
744
+ libnut.moveMouse(Math.round(from.x), Math.round(from.y));
745
+ libnut.mouseToggle('down', 'left');
746
+ await (0, utils_namespaceObject.sleep)(100);
747
+ libnut.moveMouse(Math.round(to.x), Math.round(to.y));
748
+ await (0, utils_namespaceObject.sleep)(100);
749
+ libnut.mouseToggle('up', 'left');
750
+ }
751
+ },
752
+ keyboard: {
753
+ typeText: async (value, opts)=>{
754
+ external_node_assert_default()(libnut, 'libnut not initialized');
755
+ const element = opts?.target;
756
+ if (element) {
757
+ const [x, y] = element.center;
758
+ libnut.moveMouse(Math.round(x), Math.round(y));
759
+ libnut.mouseClick('left');
760
+ await (0, utils_namespaceObject.sleep)(INPUT_FOCUS_DELAY);
761
+ if (opts?.replace !== false) {
762
+ await this.selectAllAndDelete();
763
+ await (0, utils_namespaceObject.sleep)(INPUT_CLEAR_DELAY);
764
+ }
765
+ }
766
+ await this.smartTypeString(value);
767
+ },
768
+ keyboardPress: async (keyName, opts)=>{
769
+ external_node_assert_default()(libnut, 'libnut not initialized');
770
+ const target = opts?.target;
771
+ if (target) {
772
+ const [x, y] = target.center;
773
+ libnut.moveMouse(Math.round(x), Math.round(y));
774
+ libnut.mouseClick('left');
775
+ await (0, utils_namespaceObject.sleep)(50);
776
+ }
777
+ await this.pressKeyboardShortcut(keyName);
778
+ },
779
+ clearInput: async (target)=>{
780
+ external_node_assert_default()(libnut, 'libnut not initialized');
781
+ if (target) {
782
+ const element = target;
783
+ const [x, y] = element.center;
784
+ libnut.moveMouse(Math.round(x), Math.round(y));
785
+ libnut.mouseClick('left');
786
+ await (0, utils_namespaceObject.sleep)(100);
787
+ }
788
+ await this.selectAllAndDelete();
789
+ await (0, utils_namespaceObject.sleep)(50);
790
+ }
791
+ },
792
+ scroll: {
793
+ scroll: async (param)=>{
794
+ await this.performScroll(param);
795
+ }
796
+ }
797
+ });
825
798
  this.options = options;
826
799
  this.displayId = options?.displayId;
827
800
  this.useAppleScript = 'darwin' === process.platform && options?.keyboardDriver !== 'libnut';
@@ -836,10 +809,578 @@ function createPlatformActions() {
836
809
  })
837
810
  };
838
811
  }
839
- require("node:events");
840
- require("node:readline");
841
- (0, logger_namespaceObject.getDebug)('rdp:backend');
842
- (0, logger_namespaceObject.getDebug)('rdp:device');
812
+ const external_node_events_namespaceObject = require("node:events");
813
+ const external_node_readline_namespaceObject = require("node:readline");
814
+ const platformBinaryMap = {
815
+ darwin: {
816
+ directory: 'darwin',
817
+ fileName: 'rdp-helper'
818
+ },
819
+ linux: {
820
+ directory: 'linux',
821
+ fileName: 'rdp-helper'
822
+ },
823
+ win32: {
824
+ directory: 'win32',
825
+ fileName: 'rdp-helper.exe'
826
+ }
827
+ };
828
+ function getPlatformBinary(platform) {
829
+ if (platform in platformBinaryMap) return platformBinaryMap[platform];
830
+ }
831
+ function currentDirname() {
832
+ return __dirname;
833
+ }
834
+ function getRdpHelperBinaryPath() {
835
+ const platformBinary = getPlatformBinary(process.platform);
836
+ if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
837
+ const hereDir = currentDirname();
838
+ const candidateRoots = [
839
+ (0, external_node_path_namespaceObject.resolve)(hereDir, '../..'),
840
+ (0, external_node_path_namespaceObject.resolve)(hereDir, '../../..')
841
+ ];
842
+ for (const root of candidateRoots){
843
+ const binaryPath = (0, external_node_path_namespaceObject.resolve)(root, 'bin', platformBinary.directory, platformBinary.fileName);
844
+ if ((0, external_node_fs_namespaceObject.existsSync)(binaryPath)) return binaryPath;
845
+ }
846
+ throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
847
+ }
848
+ function backend_client_define_property(obj, key, value) {
849
+ if (key in obj) Object.defineProperty(obj, key, {
850
+ value: value,
851
+ enumerable: true,
852
+ configurable: true,
853
+ writable: true
854
+ });
855
+ else obj[key] = value;
856
+ return obj;
857
+ }
858
+ const debug = (0, logger_namespaceObject.getDebug)('rdp:backend');
859
+ const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
860
+ const MAX_STDERR_LINES = 40;
861
+ class HelperProcessRDPBackendClient {
862
+ async connect(config) {
863
+ this.fatalHelperError = void 0;
864
+ await this.ensureHelperStarted();
865
+ const response = await this.send({
866
+ type: 'connect',
867
+ config
868
+ });
869
+ if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
870
+ this.connected = true;
871
+ this.fatalHelperError = void 0;
872
+ return response.info;
873
+ }
874
+ async disconnect() {
875
+ const child = this.child;
876
+ if (!child) return;
877
+ let disconnectError;
878
+ if (this.connected && null === child.exitCode) try {
879
+ const response = await this.send({
880
+ type: 'disconnect'
881
+ });
882
+ this.expectOk(response, 'disconnect');
883
+ } catch (error) {
884
+ disconnectError = error instanceof Error ? error : new Error(String(error));
885
+ }
886
+ this.connected = false;
887
+ this.fatalHelperError = void 0;
888
+ await this.shutdownHelper();
889
+ if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
890
+ }
891
+ async screenshotBase64() {
892
+ const response = await this.send({
893
+ type: 'screenshot'
894
+ });
895
+ if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
896
+ return response.base64;
897
+ }
898
+ async size() {
899
+ const response = await this.send({
900
+ type: 'size'
901
+ });
902
+ if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
903
+ return response.size;
904
+ }
905
+ async mouseMove(x, y) {
906
+ const response = await this.send({
907
+ type: 'mouseMove',
908
+ x,
909
+ y
910
+ });
911
+ this.expectOk(response, 'mouseMove');
912
+ }
913
+ async mouseButton(button, action) {
914
+ const response = await this.send({
915
+ type: 'mouseButton',
916
+ button,
917
+ action
918
+ });
919
+ this.expectOk(response, 'mouseButton');
920
+ }
921
+ async wheel(direction, amount, x, y) {
922
+ const response = await this.send({
923
+ type: 'wheel',
924
+ direction,
925
+ amount,
926
+ x,
927
+ y
928
+ });
929
+ this.expectOk(response, 'wheel');
930
+ }
931
+ async keyPress(keyName) {
932
+ const response = await this.send({
933
+ type: 'keyPress',
934
+ keyName
935
+ });
936
+ this.expectOk(response, 'keyPress');
937
+ }
938
+ async typeText(text) {
939
+ const response = await this.send({
940
+ type: 'typeText',
941
+ text
942
+ });
943
+ this.expectOk(response, 'typeText');
944
+ }
945
+ async clearInput() {
946
+ const response = await this.send({
947
+ type: 'clearInput'
948
+ });
949
+ this.expectOk(response, 'clearInput');
950
+ }
951
+ async ensureHelperStarted() {
952
+ if (this.child && null === this.child.exitCode) return;
953
+ const helperPath = this.resolveHelperPath();
954
+ debug('starting rdp helper', {
955
+ helperPath
956
+ });
957
+ const child = this.spawnFn(helperPath, [], {
958
+ stdio: [
959
+ 'pipe',
960
+ 'pipe',
961
+ 'pipe'
962
+ ]
963
+ });
964
+ child.stdout.setEncoding('utf8');
965
+ child.stderr.setEncoding('utf8');
966
+ this.child = child;
967
+ this.stderrLines.length = 0;
968
+ this.stdoutReader = (0, external_node_readline_namespaceObject.createInterface)({
969
+ input: child.stdout,
970
+ crlfDelay: 1 / 0
971
+ });
972
+ this.stderrReader = (0, external_node_readline_namespaceObject.createInterface)({
973
+ input: child.stderr,
974
+ crlfDelay: 1 / 0
975
+ });
976
+ this.stdoutReader.on('line', (line)=>{
977
+ this.handleStdoutLine(line);
978
+ });
979
+ this.stderrReader.on('line', (line)=>{
980
+ this.captureStderrLine(line);
981
+ });
982
+ child.on('exit', (code, signal)=>{
983
+ this.connected = false;
984
+ const error = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`);
985
+ this.fatalHelperError = error;
986
+ this.rejectPending(error);
987
+ this.disposeReaders();
988
+ this.child = void 0;
989
+ });
990
+ child.on('error', (error)=>{
991
+ this.connected = false;
992
+ const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`);
993
+ this.fatalHelperError = helperError;
994
+ this.rejectPending(helperError);
995
+ this.disposeReaders();
996
+ this.child = void 0;
997
+ });
998
+ }
999
+ handleStdoutLine(line) {
1000
+ if (!line.trim()) return;
1001
+ let parsed;
1002
+ try {
1003
+ parsed = JSON.parse(line);
1004
+ } catch (error) {
1005
+ const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`);
1006
+ this.rejectPending(protocolError);
1007
+ this.shutdownHelper(protocolError);
1008
+ return;
1009
+ }
1010
+ const pending = this.pending.get(parsed.id);
1011
+ if (!pending) return void debug('dropping response for unknown request id', parsed);
1012
+ this.pending.delete(parsed.id);
1013
+ if (parsed.ok) return void pending.resolve(parsed.payload);
1014
+ pending.reject(this.createHelperError(parsed.error.message, parsed.error.code));
1015
+ }
1016
+ captureStderrLine(line) {
1017
+ if (!line.trim()) return;
1018
+ this.stderrLines.push(line);
1019
+ if (this.stderrLines.length > MAX_STDERR_LINES) this.stderrLines.shift();
1020
+ }
1021
+ async send(payload) {
1022
+ if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
1023
+ await this.ensureHelperStarted();
1024
+ const child = this.child;
1025
+ if (!child || null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
1026
+ const id = `req-${++this.nextRequestId}`;
1027
+ const request = {
1028
+ id,
1029
+ payload
1030
+ };
1031
+ return new Promise((resolve, reject)=>{
1032
+ this.pending.set(id, {
1033
+ resolve,
1034
+ reject
1035
+ });
1036
+ child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
1037
+ if (!error) return;
1038
+ this.pending.delete(id);
1039
+ reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`));
1040
+ });
1041
+ });
1042
+ }
1043
+ expectOk(response, actionName) {
1044
+ if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
1045
+ }
1046
+ rejectPending(error) {
1047
+ for (const { reject } of this.pending.values())reject(error);
1048
+ this.pending.clear();
1049
+ }
1050
+ createHelperError(message, code) {
1051
+ const stderrSummary = this.stderrLines.join('\n').trim();
1052
+ const suffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
1053
+ const error = new Error(`${message}${suffix}`);
1054
+ if (code) error.name = code;
1055
+ return error;
1056
+ }
1057
+ disposeReaders() {
1058
+ this.stdoutReader?.close();
1059
+ this.stderrReader?.close();
1060
+ this.stdoutReader = void 0;
1061
+ this.stderrReader = void 0;
1062
+ }
1063
+ async shutdownHelper(rootError) {
1064
+ const child = this.child;
1065
+ this.child = void 0;
1066
+ this.disposeReaders();
1067
+ if (!child) return;
1068
+ this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
1069
+ if (null !== child.exitCode) return;
1070
+ child.stdin.end();
1071
+ const exited = Promise.race([
1072
+ (0, external_node_events_namespaceObject.once)(child, 'exit'),
1073
+ new Promise((resolve)=>{
1074
+ setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1075
+ })
1076
+ ]);
1077
+ const result = await exited;
1078
+ if ('timeout' !== result) return;
1079
+ child.kill('SIGTERM');
1080
+ const terminated = Promise.race([
1081
+ (0, external_node_events_namespaceObject.once)(child, 'exit'),
1082
+ new Promise((resolve)=>{
1083
+ setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1084
+ })
1085
+ ]);
1086
+ const terminateResult = await terminated;
1087
+ if ('timeout' !== terminateResult) return;
1088
+ child.kill('SIGKILL');
1089
+ await (0, external_node_events_namespaceObject.once)(child, 'exit');
1090
+ }
1091
+ constructor(options){
1092
+ backend_client_define_property(this, "spawnFn", void 0);
1093
+ backend_client_define_property(this, "resolveHelperPath", void 0);
1094
+ backend_client_define_property(this, "child", void 0);
1095
+ backend_client_define_property(this, "stdoutReader", void 0);
1096
+ backend_client_define_property(this, "stderrReader", void 0);
1097
+ backend_client_define_property(this, "pending", new Map());
1098
+ backend_client_define_property(this, "stderrLines", []);
1099
+ backend_client_define_property(this, "nextRequestId", 0);
1100
+ backend_client_define_property(this, "connected", false);
1101
+ backend_client_define_property(this, "fatalHelperError", void 0);
1102
+ this.spawnFn = options?.spawnFn || external_node_child_process_namespaceObject.spawn;
1103
+ const overridePath = options?.helperPath;
1104
+ this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
1105
+ }
1106
+ }
1107
+ function createDefaultRDPBackendClient() {
1108
+ return new HelperProcessRDPBackendClient();
1109
+ }
1110
+ function device_define_property(obj, key, value) {
1111
+ if (key in obj) Object.defineProperty(obj, key, {
1112
+ value: value,
1113
+ enumerable: true,
1114
+ configurable: true,
1115
+ writable: true
1116
+ });
1117
+ else obj[key] = value;
1118
+ return obj;
1119
+ }
1120
+ const device_debug = (0, logger_namespaceObject.getDebug)('rdp:device');
1121
+ const device_SMOOTH_MOVE_STEPS_TAP = 8;
1122
+ const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
1123
+ const SMOOTH_MOVE_STEPS_DRAG = 12;
1124
+ const device_SMOOTH_MOVE_DELAY_TAP = 8;
1125
+ const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
1126
+ const SMOOTH_MOVE_DELAY_DRAG = 10;
1127
+ const device_MOUSE_MOVE_EFFECT_WAIT = 300;
1128
+ const device_CLICK_HOLD_DURATION = 50;
1129
+ const DRAG_HOLD_DURATION = 100;
1130
+ const device_INPUT_FOCUS_DELAY = 300;
1131
+ const device_INPUT_CLEAR_DELAY = 150;
1132
+ const device_SCROLL_STEP_DELAY = 100;
1133
+ const device_SCROLL_COMPLETE_DELAY = 500;
1134
+ const DEFAULT_SCROLL_DISTANCE = 480;
1135
+ const device_EDGE_SCROLL_STEPS = 10;
1136
+ const DEFAULT_SCROLL_STEP_AMOUNT = 120;
1137
+ class RDPDevice {
1138
+ describe() {
1139
+ const port = this.options.port || 3389;
1140
+ const username = this.options.username ? ` as ${this.options.username}` : '';
1141
+ const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
1142
+ return `RDP Device ${this.options.host}:${port}${username}${session}`;
1143
+ }
1144
+ async connect() {
1145
+ this.throwIfDestroyed();
1146
+ device_debug('connecting to rdp backend', {
1147
+ host: this.options.host,
1148
+ port: this.options.port,
1149
+ username: this.options.username
1150
+ });
1151
+ this.connectionInfo = await this.backend.connect(this.options);
1152
+ this.cursorPosition = [
1153
+ Math.round(this.connectionInfo.size.width / 2),
1154
+ Math.round(this.connectionInfo.size.height / 2)
1155
+ ];
1156
+ }
1157
+ async screenshotBase64() {
1158
+ this.assertConnected();
1159
+ return this.backend.screenshotBase64();
1160
+ }
1161
+ async size() {
1162
+ this.assertConnected();
1163
+ return this.backend.size();
1164
+ }
1165
+ async destroy() {
1166
+ if (this.destroyed) return;
1167
+ this.destroyed = true;
1168
+ this.connectionInfo = void 0;
1169
+ this.cursorPosition = void 0;
1170
+ await this.backend.disconnect();
1171
+ }
1172
+ actionSpace() {
1173
+ const defaultActions = [
1174
+ ...(0, device_namespaceObject.defineActionsFromInputPrimitives)(this.inputPrimitives),
1175
+ (0, device_namespaceObject.defineAction)({
1176
+ name: 'ListDisplays',
1177
+ description: 'List all available displays/monitors',
1178
+ call: async ()=>{
1179
+ this.assertConnected();
1180
+ const size = await this.size();
1181
+ return [
1182
+ {
1183
+ id: this.connectionInfo?.sessionId || this.options.host,
1184
+ name: `RDP ${this.connectionInfo?.server || this.options.host} (${size.width}x${size.height})`,
1185
+ primary: true
1186
+ }
1187
+ ];
1188
+ }
1189
+ })
1190
+ ];
1191
+ return [
1192
+ ...defaultActions,
1193
+ ...this.options.customActions || []
1194
+ ];
1195
+ }
1196
+ assertConnected() {
1197
+ this.throwIfDestroyed();
1198
+ if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
1199
+ }
1200
+ throwIfDestroyed() {
1201
+ if (this.destroyed) throw new Error('RDPDevice has been destroyed');
1202
+ }
1203
+ async moveToElement(element, options) {
1204
+ this.assertConnected();
1205
+ const targetX = Math.round(element.center[0]);
1206
+ const targetY = Math.round(element.center[1]);
1207
+ await this.movePointer(targetX, targetY, options);
1208
+ }
1209
+ async clearInput() {
1210
+ if (this.backend.clearInput) return void await this.backend.clearInput();
1211
+ await this.backend.keyPress('Control+A');
1212
+ await this.backend.keyPress('Backspace');
1213
+ }
1214
+ edgeScrollDirection(scrollType) {
1215
+ switch(scrollType){
1216
+ case 'scrollToTop':
1217
+ return 'up';
1218
+ case 'scrollToBottom':
1219
+ return 'down';
1220
+ case 'scrollToLeft':
1221
+ return 'left';
1222
+ case 'scrollToRight':
1223
+ return 'right';
1224
+ case 'singleAction':
1225
+ return 'down';
1226
+ default:
1227
+ throw new Error(`Unsupported scroll type: ${scrollType}`);
1228
+ }
1229
+ }
1230
+ async movePointer(targetX, targetY, options) {
1231
+ this.assertConnected();
1232
+ const start = this.cursorPosition || [
1233
+ targetX,
1234
+ targetY
1235
+ ];
1236
+ const steps = Math.max(1, options?.steps || 1);
1237
+ const stepDelayMs = options?.stepDelayMs || 0;
1238
+ for(let step = 1; step <= steps; step++){
1239
+ const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
1240
+ const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
1241
+ await this.backend.mouseMove(x, y);
1242
+ this.cursorPosition = [
1243
+ x,
1244
+ y
1245
+ ];
1246
+ if (stepDelayMs > 0 && step < steps) await (0, utils_namespaceObject.sleep)(stepDelayMs);
1247
+ }
1248
+ if (options?.settleDelayMs) await (0, utils_namespaceObject.sleep)(options.settleDelayMs);
1249
+ }
1250
+ async performWheel(direction, amount, x, y) {
1251
+ let remaining = Math.abs(amount);
1252
+ if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
1253
+ while(remaining > 0){
1254
+ const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
1255
+ await this.backend.wheel(direction, chunk, x, y);
1256
+ remaining -= chunk;
1257
+ if (remaining > 0) await (0, utils_namespaceObject.sleep)(device_SCROLL_STEP_DELAY);
1258
+ }
1259
+ }
1260
+ constructor(options){
1261
+ device_define_property(this, "interfaceType", 'rdp');
1262
+ device_define_property(this, "options", void 0);
1263
+ device_define_property(this, "backend", void 0);
1264
+ device_define_property(this, "connectionInfo", void 0);
1265
+ device_define_property(this, "destroyed", false);
1266
+ device_define_property(this, "cursorPosition", void 0);
1267
+ device_define_property(this, "uri", void 0);
1268
+ device_define_property(this, "inputPrimitives", {
1269
+ pointer: {
1270
+ tap: async ({ x, y })=>{
1271
+ await this.movePointer(Math.round(x), Math.round(y), {
1272
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1273
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1274
+ });
1275
+ await this.backend.mouseButton('left', 'down');
1276
+ await (0, utils_namespaceObject.sleep)(device_CLICK_HOLD_DURATION);
1277
+ await this.backend.mouseButton('left', 'up');
1278
+ },
1279
+ doubleClick: async ({ x, y })=>{
1280
+ await this.movePointer(Math.round(x), Math.round(y), {
1281
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1282
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1283
+ });
1284
+ await this.backend.mouseButton('left', 'doubleClick');
1285
+ },
1286
+ rightClick: async ({ x, y })=>{
1287
+ await this.movePointer(Math.round(x), Math.round(y), {
1288
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1289
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1290
+ });
1291
+ await this.backend.mouseButton('right', 'click');
1292
+ },
1293
+ hover: async ({ x, y })=>{
1294
+ await this.movePointer(Math.round(x), Math.round(y), {
1295
+ steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1296
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
1297
+ settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
1298
+ });
1299
+ },
1300
+ dragAndDrop: async (from, to)=>{
1301
+ await this.movePointer(Math.round(from.x), Math.round(from.y), {
1302
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1303
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1304
+ });
1305
+ await this.backend.mouseButton('left', 'down');
1306
+ await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
1307
+ await this.movePointer(Math.round(to.x), Math.round(to.y), {
1308
+ steps: SMOOTH_MOVE_STEPS_DRAG,
1309
+ stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
1310
+ });
1311
+ await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
1312
+ await this.backend.mouseButton('left', 'up');
1313
+ }
1314
+ },
1315
+ keyboard: {
1316
+ typeText: async (value, opts)=>{
1317
+ this.assertConnected();
1318
+ const target = opts?.target;
1319
+ if (target) {
1320
+ await this.inputPrimitives.pointer.tap({
1321
+ x: target.center[0],
1322
+ y: target.center[1]
1323
+ });
1324
+ await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
1325
+ }
1326
+ if (opts?.replace !== false) {
1327
+ await this.clearInput();
1328
+ await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
1329
+ }
1330
+ if (opts?.focusOnly || !value) return;
1331
+ await this.backend.typeText(value);
1332
+ },
1333
+ clearInput: async (target)=>{
1334
+ this.assertConnected();
1335
+ const element = target;
1336
+ if (element) {
1337
+ await this.inputPrimitives.pointer.tap({
1338
+ x: element.center[0],
1339
+ y: element.center[1]
1340
+ });
1341
+ await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
1342
+ }
1343
+ await this.clearInput();
1344
+ await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
1345
+ },
1346
+ keyboardPress: async (keyName, opts)=>{
1347
+ this.assertConnected();
1348
+ const target = opts?.target;
1349
+ if (target) await this.inputPrimitives.pointer.tap({
1350
+ x: target.center[0],
1351
+ y: target.center[1]
1352
+ });
1353
+ await this.backend.keyPress(keyName);
1354
+ }
1355
+ },
1356
+ scroll: {
1357
+ scroll: async (param)=>{
1358
+ this.assertConnected();
1359
+ const target = param.locate;
1360
+ if (target) await this.moveToElement(target, {
1361
+ steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1362
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
1363
+ });
1364
+ if (param.scrollType && 'singleAction' !== param.scrollType) {
1365
+ const direction = this.edgeScrollDirection(param.scrollType);
1366
+ for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
1367
+ await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
1368
+ return;
1369
+ }
1370
+ await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
1371
+ await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
1372
+ }
1373
+ }
1374
+ });
1375
+ this.options = {
1376
+ port: 3389,
1377
+ securityProtocol: 'auto',
1378
+ ignoreCertificate: false,
1379
+ ...options
1380
+ };
1381
+ this.backend = options.backend || createDefaultRDPBackendClient();
1382
+ }
1383
+ }
843
1384
  class ComputerAgent extends agent_namespaceObject.Agent {
844
1385
  }
845
1386
  function createLocalComputerDevice(opts) {
@@ -851,12 +1392,33 @@ function createLocalComputerDevice(opts) {
851
1392
  xvfbResolution: opts?.xvfbResolution
852
1393
  });
853
1394
  }
1395
+ function createRDPComputerDevice(opts) {
1396
+ return new RDPDevice({
1397
+ host: opts.host,
1398
+ port: opts.port,
1399
+ username: opts.username,
1400
+ password: opts.password,
1401
+ domain: opts.domain,
1402
+ adminSession: opts.adminSession,
1403
+ ignoreCertificate: opts.ignoreCertificate,
1404
+ securityProtocol: opts.securityProtocol,
1405
+ desktopWidth: opts.desktopWidth,
1406
+ desktopHeight: opts.desktopHeight,
1407
+ backend: opts.backend,
1408
+ customActions: opts.customActions
1409
+ });
1410
+ }
854
1411
  async function agentForComputer(opts) {
855
1412
  const device = createLocalComputerDevice(opts);
856
1413
  await device.connect();
857
1414
  return new ComputerAgent(device, opts);
858
1415
  }
859
1416
  const agentFromComputer = agentForComputer;
1417
+ async function agentForRDPComputer(opts) {
1418
+ const device = createRDPComputerDevice(opts);
1419
+ await device.connect();
1420
+ return new ComputerAgent(device, opts);
1421
+ }
860
1422
  function mcp_tools_define_property(obj, key, value) {
861
1423
  if (key in obj) Object.defineProperty(obj, key, {
862
1424
  value: value,
@@ -868,10 +1430,61 @@ function mcp_tools_define_property(obj, key, value) {
868
1430
  return obj;
869
1431
  }
870
1432
  const mcp_tools_debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
1433
+ const RDP_SECURITY_PROTOCOLS = [
1434
+ 'auto',
1435
+ 'tls',
1436
+ 'nla',
1437
+ 'rdp'
1438
+ ];
871
1439
  const computerInitArgShape = {
872
- displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
873
- headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
1440
+ displayId: core_namespaceObject.z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
1441
+ headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
1442
+ host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
1443
+ port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
1444
+ username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
1445
+ password: core_namespaceObject.z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
1446
+ domain: core_namespaceObject.z.string().optional().describe('RDP domain. Requires host.'),
1447
+ adminSession: core_namespaceObject.z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
1448
+ ignoreCertificate: core_namespaceObject.z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
1449
+ securityProtocol: core_namespaceObject.z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
1450
+ desktopWidth: core_namespaceObject.z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
1451
+ desktopHeight: core_namespaceObject.z.number().optional().describe('Remote desktop height in pixels. Requires host.')
874
1452
  };
1453
+ function adaptComputerInitArgs(extracted) {
1454
+ if (!extracted || 0 === Object.keys(extracted).length) return;
1455
+ if (extracted.host) {
1456
+ const { displayId: _d, headless: _h, ...rdpFields } = extracted;
1457
+ return {
1458
+ mode: 'rdp',
1459
+ ...rdpFields,
1460
+ host: extracted.host
1461
+ };
1462
+ }
1463
+ return {
1464
+ mode: 'local',
1465
+ displayId: extracted.displayId,
1466
+ headless: extracted.headless
1467
+ };
1468
+ }
1469
+ function shouldRetargetAgent(opts) {
1470
+ if (!opts) return false;
1471
+ if ('rdp' === opts.mode) return true;
1472
+ return void 0 !== opts.displayId || void 0 !== opts.headless;
1473
+ }
1474
+ function describeConnectTarget(opts) {
1475
+ if (opts?.mode === 'rdp') {
1476
+ const portSuffix = opts.port ? `:${opts.port}` : '';
1477
+ const userSuffix = opts.username ? ` as ${opts.username}` : '';
1478
+ return ` via RDP (${opts.host}${portSuffix}${userSuffix})`;
1479
+ }
1480
+ if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
1481
+ return ' (Primary display)';
1482
+ }
1483
+ function getCliReportSessionTarget(opts) {
1484
+ if (opts?.mode === 'rdp') return `rdp:${opts.host}`;
1485
+ if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
1486
+ return 'primary';
1487
+ }
875
1488
  class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
876
1489
  getCliReportSessionName() {
877
1490
  return 'midscene-computer';
@@ -880,9 +1493,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
880
1493
  return new ComputerDevice({});
881
1494
  }
882
1495
  async ensureAgent(opts) {
883
- const displayId = opts?.displayId;
884
- const headless = opts?.headless;
885
- if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
1496
+ if (this.agent && shouldRetargetAgent(opts)) {
886
1497
  try {
887
1498
  await this.agent.destroy?.();
888
1499
  } catch (error) {
@@ -891,8 +1502,20 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
891
1502
  this.agent = void 0;
892
1503
  }
893
1504
  if (this.agent) return this.agent;
894
- mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
895
1505
  const reportOptions = this.readCliReportAgentOptions();
1506
+ if (opts?.mode === 'rdp') {
1507
+ mcp_tools_debug('Creating RDP Computer agent for host:', opts.host);
1508
+ const { mode: _mode, ...rdpFields } = opts;
1509
+ const agent = await agentForRDPComputer({
1510
+ ...rdpFields,
1511
+ ...reportOptions ?? {}
1512
+ });
1513
+ this.agent = agent;
1514
+ return agent;
1515
+ }
1516
+ const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
1517
+ const headless = opts?.mode === 'local' ? opts.headless : void 0;
1518
+ mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
896
1519
  const agentOpts = {
897
1520
  ...displayId ? {
898
1521
  displayId
@@ -910,12 +1533,12 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
910
1533
  return [
911
1534
  {
912
1535
  name: 'computer_connect',
913
- description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
1536
+ description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
914
1537
  schema: this.getAgentInitArgSchema(),
915
1538
  cli: this.getAgentInitArgCliMetadata(),
916
1539
  handler: async (args)=>{
917
1540
  const initArgs = this.extractAgentInitParam(args);
918
- const reportSession = this.createNewCliReportSession(initArgs?.displayId ?? 'primary');
1541
+ const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
919
1542
  this.commitCliReportSession(reportSession);
920
1543
  if (this.agent) {
921
1544
  try {
@@ -931,7 +1554,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
931
1554
  content: [
932
1555
  {
933
1556
  type: 'text',
934
- text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
1557
+ text: `Connected to computer${describeConnectTarget(initArgs)}`
935
1558
  },
936
1559
  ...this.buildScreenshotContent(screenshot)
937
1560
  ]
@@ -969,14 +1592,14 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
969
1592
  cli: {
970
1593
  preferBareKeys: true
971
1594
  },
972
- adapt: (extracted)=>extracted
1595
+ adapt: (extracted)=>adaptComputerInitArgs(extracted)
973
1596
  });
974
1597
  }
975
1598
  }
976
1599
  const tools = new ComputerMidsceneTools();
977
1600
  (0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-computer', {
978
1601
  stripPrefix: 'computer_',
979
- version: "1.8.0",
1602
+ version: "1.8.1",
980
1603
  extraCommands: (0, core_namespaceObject.createReportCliCommands)()
981
1604
  }).catch((e)=>{
982
1605
  process.exit((0, cli_namespaceObject.reportCLIError)(e));