@midscene/computer 1.8.0 → 1.8.1-beta-20260513084557.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -452,7 +452,7 @@ Available Displays: ${displays.length > 0 ? displays.map((d)=>d.name).join(', ')
452
452
  }
453
453
  async healthCheck() {
454
454
  console.log('[HealthCheck] Starting health check...');
455
- console.log("[HealthCheck] @midscene/computer v1.8.0");
455
+ console.log("[HealthCheck] @midscene/computer v1.8.1-beta-20260513084557.0");
456
456
  console.log('[HealthCheck] Taking screenshot...');
457
457
  const screenshotTimeout = 15000;
458
458
  let timeoutId;
@@ -851,10 +851,600 @@ function createPlatformActions() {
851
851
  })
852
852
  };
853
853
  }
854
- require("node:events");
855
- require("node:readline");
856
- (0, logger_namespaceObject.getDebug)('rdp:backend');
857
- (0, logger_namespaceObject.getDebug)('rdp:device');
854
+ const external_node_events_namespaceObject = require("node:events");
855
+ const external_node_readline_namespaceObject = require("node:readline");
856
+ const platformBinaryMap = {
857
+ darwin: {
858
+ directory: 'darwin',
859
+ fileName: 'rdp-helper'
860
+ },
861
+ linux: {
862
+ directory: 'linux',
863
+ fileName: 'rdp-helper'
864
+ },
865
+ win32: {
866
+ directory: 'win32',
867
+ fileName: 'rdp-helper.exe'
868
+ }
869
+ };
870
+ function getPlatformBinary(platform) {
871
+ if (platform in platformBinaryMap) return platformBinaryMap[platform];
872
+ }
873
+ function currentDirname() {
874
+ return __dirname;
875
+ }
876
+ function getRdpHelperBinaryPath() {
877
+ const platformBinary = getPlatformBinary(process.platform);
878
+ if (!platformBinary) throw new Error(`@midscene/computer RDP helper does not support platform ${process.platform}`);
879
+ const hereDir = currentDirname();
880
+ const candidateRoots = [
881
+ (0, external_node_path_namespaceObject.resolve)(hereDir, '../..'),
882
+ (0, external_node_path_namespaceObject.resolve)(hereDir, '../../..')
883
+ ];
884
+ for (const root of candidateRoots){
885
+ const binaryPath = (0, external_node_path_namespaceObject.resolve)(root, 'bin', platformBinary.directory, platformBinary.fileName);
886
+ if ((0, external_node_fs_namespaceObject.existsSync)(binaryPath)) return binaryPath;
887
+ }
888
+ throw new Error(`RDP helper binary not found for ${process.platform}. Run \`pnpm --filter @midscene/computer run build:native\` first.`);
889
+ }
890
+ function backend_client_define_property(obj, key, value) {
891
+ if (key in obj) Object.defineProperty(obj, key, {
892
+ value: value,
893
+ enumerable: true,
894
+ configurable: true,
895
+ writable: true
896
+ });
897
+ else obj[key] = value;
898
+ return obj;
899
+ }
900
+ const debug = (0, logger_namespaceObject.getDebug)('rdp:backend');
901
+ const HELPER_SHUTDOWN_TIMEOUT_MS = 3000;
902
+ const MAX_STDERR_LINES = 40;
903
+ class HelperProcessRDPBackendClient {
904
+ async connect(config) {
905
+ this.fatalHelperError = void 0;
906
+ await this.ensureHelperStarted();
907
+ const response = await this.send({
908
+ type: 'connect',
909
+ config
910
+ });
911
+ if ('connected' !== response.type) throw new Error(`Expected connected response, got ${response.type}`);
912
+ this.connected = true;
913
+ this.fatalHelperError = void 0;
914
+ return response.info;
915
+ }
916
+ async disconnect() {
917
+ const child = this.child;
918
+ if (!child) return;
919
+ let disconnectError;
920
+ if (this.connected && null === child.exitCode) try {
921
+ const response = await this.send({
922
+ type: 'disconnect'
923
+ });
924
+ this.expectOk(response, 'disconnect');
925
+ } catch (error) {
926
+ disconnectError = error instanceof Error ? error : new Error(String(error));
927
+ }
928
+ this.connected = false;
929
+ this.fatalHelperError = void 0;
930
+ await this.shutdownHelper();
931
+ if (disconnectError && !/RDP helper exited unexpectedly|RDP helper is not running|RDP helper shut down/u.test(disconnectError.message)) throw disconnectError;
932
+ }
933
+ async screenshotBase64() {
934
+ const response = await this.send({
935
+ type: 'screenshot'
936
+ });
937
+ if ('screenshot' !== response.type) throw new Error(`Expected screenshot response, got ${response.type}`);
938
+ return response.base64;
939
+ }
940
+ async size() {
941
+ const response = await this.send({
942
+ type: 'size'
943
+ });
944
+ if ('size' !== response.type) throw new Error(`Expected size response, got ${response.type}`);
945
+ return response.size;
946
+ }
947
+ async mouseMove(x, y) {
948
+ const response = await this.send({
949
+ type: 'mouseMove',
950
+ x,
951
+ y
952
+ });
953
+ this.expectOk(response, 'mouseMove');
954
+ }
955
+ async mouseButton(button, action) {
956
+ const response = await this.send({
957
+ type: 'mouseButton',
958
+ button,
959
+ action
960
+ });
961
+ this.expectOk(response, 'mouseButton');
962
+ }
963
+ async wheel(direction, amount, x, y) {
964
+ const response = await this.send({
965
+ type: 'wheel',
966
+ direction,
967
+ amount,
968
+ x,
969
+ y
970
+ });
971
+ this.expectOk(response, 'wheel');
972
+ }
973
+ async keyPress(keyName) {
974
+ const response = await this.send({
975
+ type: 'keyPress',
976
+ keyName
977
+ });
978
+ this.expectOk(response, 'keyPress');
979
+ }
980
+ async typeText(text) {
981
+ const response = await this.send({
982
+ type: 'typeText',
983
+ text
984
+ });
985
+ this.expectOk(response, 'typeText');
986
+ }
987
+ async clearInput() {
988
+ const response = await this.send({
989
+ type: 'clearInput'
990
+ });
991
+ this.expectOk(response, 'clearInput');
992
+ }
993
+ async ensureHelperStarted() {
994
+ if (this.child && null === this.child.exitCode) return;
995
+ const helperPath = this.resolveHelperPath();
996
+ debug('starting rdp helper', {
997
+ helperPath
998
+ });
999
+ const child = this.spawnFn(helperPath, [], {
1000
+ stdio: [
1001
+ 'pipe',
1002
+ 'pipe',
1003
+ 'pipe'
1004
+ ]
1005
+ });
1006
+ child.stdout.setEncoding('utf8');
1007
+ child.stderr.setEncoding('utf8');
1008
+ this.child = child;
1009
+ this.stderrLines.length = 0;
1010
+ this.stdoutReader = (0, external_node_readline_namespaceObject.createInterface)({
1011
+ input: child.stdout,
1012
+ crlfDelay: 1 / 0
1013
+ });
1014
+ this.stderrReader = (0, external_node_readline_namespaceObject.createInterface)({
1015
+ input: child.stderr,
1016
+ crlfDelay: 1 / 0
1017
+ });
1018
+ this.stdoutReader.on('line', (line)=>{
1019
+ this.handleStdoutLine(line);
1020
+ });
1021
+ this.stderrReader.on('line', (line)=>{
1022
+ this.captureStderrLine(line);
1023
+ });
1024
+ child.on('exit', (code, signal)=>{
1025
+ this.connected = false;
1026
+ const error = this.createHelperError(`RDP helper exited unexpectedly (code=${code}, signal=${signal})`);
1027
+ this.fatalHelperError = error;
1028
+ this.rejectPending(error);
1029
+ this.disposeReaders();
1030
+ this.child = void 0;
1031
+ });
1032
+ child.on('error', (error)=>{
1033
+ this.connected = false;
1034
+ const helperError = this.createHelperError(`Failed to start RDP helper: ${error.message}`);
1035
+ this.fatalHelperError = helperError;
1036
+ this.rejectPending(helperError);
1037
+ this.disposeReaders();
1038
+ this.child = void 0;
1039
+ });
1040
+ }
1041
+ handleStdoutLine(line) {
1042
+ if (!line.trim()) return;
1043
+ let parsed;
1044
+ try {
1045
+ parsed = JSON.parse(line);
1046
+ } catch (error) {
1047
+ const protocolError = this.createHelperError(`RDP helper emitted malformed JSON: ${line}`);
1048
+ this.rejectPending(protocolError);
1049
+ this.shutdownHelper(protocolError);
1050
+ return;
1051
+ }
1052
+ const pending = this.pending.get(parsed.id);
1053
+ if (!pending) return void debug('dropping response for unknown request id', parsed);
1054
+ this.pending.delete(parsed.id);
1055
+ if (parsed.ok) return void pending.resolve(parsed.payload);
1056
+ pending.reject(this.createHelperError(parsed.error.message, parsed.error.code));
1057
+ }
1058
+ captureStderrLine(line) {
1059
+ if (!line.trim()) return;
1060
+ this.stderrLines.push(line);
1061
+ if (this.stderrLines.length > MAX_STDERR_LINES) this.stderrLines.shift();
1062
+ }
1063
+ async send(payload) {
1064
+ if ('connect' !== payload.type && this.fatalHelperError && (!this.child || null !== this.child.exitCode)) throw this.fatalHelperError;
1065
+ await this.ensureHelperStarted();
1066
+ const child = this.child;
1067
+ if (!child || null !== child.exitCode) throw this.createHelperError('RDP helper is not running');
1068
+ const id = `req-${++this.nextRequestId}`;
1069
+ const request = {
1070
+ id,
1071
+ payload
1072
+ };
1073
+ return new Promise((resolve, reject)=>{
1074
+ this.pending.set(id, {
1075
+ resolve,
1076
+ reject
1077
+ });
1078
+ child.stdin.write(`${JSON.stringify(request)}\n`, (error)=>{
1079
+ if (!error) return;
1080
+ this.pending.delete(id);
1081
+ reject(this.createHelperError(`Failed to send ${payload.type} request to RDP helper: ${error.message}`));
1082
+ });
1083
+ });
1084
+ }
1085
+ expectOk(response, actionName) {
1086
+ if ('ok' !== response.type) throw new Error(`Expected ok response for ${actionName}, got ${response.type}`);
1087
+ }
1088
+ rejectPending(error) {
1089
+ for (const { reject } of this.pending.values())reject(error);
1090
+ this.pending.clear();
1091
+ }
1092
+ createHelperError(message, code) {
1093
+ const stderrSummary = this.stderrLines.join('\n').trim();
1094
+ const suffix = stderrSummary ? `\nHelper stderr:\n${stderrSummary}` : '';
1095
+ const error = new Error(`${message}${suffix}`);
1096
+ if (code) error.name = code;
1097
+ return error;
1098
+ }
1099
+ disposeReaders() {
1100
+ this.stdoutReader?.close();
1101
+ this.stderrReader?.close();
1102
+ this.stdoutReader = void 0;
1103
+ this.stderrReader = void 0;
1104
+ }
1105
+ async shutdownHelper(rootError) {
1106
+ const child = this.child;
1107
+ this.child = void 0;
1108
+ this.disposeReaders();
1109
+ if (!child) return;
1110
+ this.rejectPending(rootError || this.createHelperError('RDP helper shut down'));
1111
+ if (null !== child.exitCode) return;
1112
+ child.stdin.end();
1113
+ const exited = Promise.race([
1114
+ (0, external_node_events_namespaceObject.once)(child, 'exit'),
1115
+ new Promise((resolve)=>{
1116
+ setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1117
+ })
1118
+ ]);
1119
+ const result = await exited;
1120
+ if ('timeout' !== result) return;
1121
+ child.kill('SIGTERM');
1122
+ const terminated = Promise.race([
1123
+ (0, external_node_events_namespaceObject.once)(child, 'exit'),
1124
+ new Promise((resolve)=>{
1125
+ setTimeout(()=>resolve('timeout'), HELPER_SHUTDOWN_TIMEOUT_MS);
1126
+ })
1127
+ ]);
1128
+ const terminateResult = await terminated;
1129
+ if ('timeout' !== terminateResult) return;
1130
+ child.kill('SIGKILL');
1131
+ await (0, external_node_events_namespaceObject.once)(child, 'exit');
1132
+ }
1133
+ constructor(options){
1134
+ backend_client_define_property(this, "spawnFn", void 0);
1135
+ backend_client_define_property(this, "resolveHelperPath", void 0);
1136
+ backend_client_define_property(this, "child", void 0);
1137
+ backend_client_define_property(this, "stdoutReader", void 0);
1138
+ backend_client_define_property(this, "stderrReader", void 0);
1139
+ backend_client_define_property(this, "pending", new Map());
1140
+ backend_client_define_property(this, "stderrLines", []);
1141
+ backend_client_define_property(this, "nextRequestId", 0);
1142
+ backend_client_define_property(this, "connected", false);
1143
+ backend_client_define_property(this, "fatalHelperError", void 0);
1144
+ this.spawnFn = options?.spawnFn || external_node_child_process_namespaceObject.spawn;
1145
+ const overridePath = options?.helperPath;
1146
+ this.resolveHelperPath = overridePath ? ()=>overridePath : getRdpHelperBinaryPath;
1147
+ }
1148
+ }
1149
+ function createDefaultRDPBackendClient() {
1150
+ return new HelperProcessRDPBackendClient();
1151
+ }
1152
+ function device_define_property(obj, key, value) {
1153
+ if (key in obj) Object.defineProperty(obj, key, {
1154
+ value: value,
1155
+ enumerable: true,
1156
+ configurable: true,
1157
+ writable: true
1158
+ });
1159
+ else obj[key] = value;
1160
+ return obj;
1161
+ }
1162
+ const device_debug = (0, logger_namespaceObject.getDebug)('rdp:device');
1163
+ const device_SMOOTH_MOVE_STEPS_TAP = 8;
1164
+ const device_SMOOTH_MOVE_STEPS_MOUSE_MOVE = 10;
1165
+ const SMOOTH_MOVE_STEPS_DRAG = 12;
1166
+ const device_SMOOTH_MOVE_DELAY_TAP = 8;
1167
+ const device_SMOOTH_MOVE_DELAY_MOUSE_MOVE = 10;
1168
+ const SMOOTH_MOVE_DELAY_DRAG = 10;
1169
+ const device_MOUSE_MOVE_EFFECT_WAIT = 300;
1170
+ const device_CLICK_HOLD_DURATION = 50;
1171
+ const DRAG_HOLD_DURATION = 100;
1172
+ const device_INPUT_FOCUS_DELAY = 300;
1173
+ const device_INPUT_CLEAR_DELAY = 150;
1174
+ const device_SCROLL_STEP_DELAY = 100;
1175
+ const device_SCROLL_COMPLETE_DELAY = 500;
1176
+ const DEFAULT_SCROLL_DISTANCE = 480;
1177
+ const device_EDGE_SCROLL_STEPS = 10;
1178
+ const DEFAULT_SCROLL_STEP_AMOUNT = 120;
1179
+ class RDPDevice {
1180
+ describe() {
1181
+ const port = this.options.port || 3389;
1182
+ const username = this.options.username ? ` as ${this.options.username}` : '';
1183
+ const session = this.connectionInfo?.sessionId ? ` [session ${this.connectionInfo.sessionId}]` : '';
1184
+ return `RDP Device ${this.options.host}:${port}${username}${session}`;
1185
+ }
1186
+ async connect() {
1187
+ this.throwIfDestroyed();
1188
+ device_debug('connecting to rdp backend', {
1189
+ host: this.options.host,
1190
+ port: this.options.port,
1191
+ username: this.options.username
1192
+ });
1193
+ this.connectionInfo = await this.backend.connect(this.options);
1194
+ this.cursorPosition = [
1195
+ Math.round(this.connectionInfo.size.width / 2),
1196
+ Math.round(this.connectionInfo.size.height / 2)
1197
+ ];
1198
+ }
1199
+ async screenshotBase64() {
1200
+ this.assertConnected();
1201
+ return this.backend.screenshotBase64();
1202
+ }
1203
+ async size() {
1204
+ this.assertConnected();
1205
+ return this.backend.size();
1206
+ }
1207
+ async destroy() {
1208
+ if (this.destroyed) return;
1209
+ this.destroyed = true;
1210
+ this.connectionInfo = void 0;
1211
+ this.cursorPosition = void 0;
1212
+ await this.backend.disconnect();
1213
+ }
1214
+ actionSpace() {
1215
+ const defaultActions = [
1216
+ (0, device_namespaceObject.defineActionTap)(async ({ locate })=>{
1217
+ const element = this.requireLocate(locate, 'tap');
1218
+ await this.moveToElement(element, {
1219
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1220
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1221
+ });
1222
+ await this.backend.mouseButton('left', 'down');
1223
+ await (0, utils_namespaceObject.sleep)(device_CLICK_HOLD_DURATION);
1224
+ await this.backend.mouseButton('left', 'up');
1225
+ }),
1226
+ (0, device_namespaceObject.defineActionDoubleClick)(async ({ locate })=>{
1227
+ const element = this.requireLocate(locate, 'double click');
1228
+ await this.moveToElement(element, {
1229
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1230
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1231
+ });
1232
+ await this.backend.mouseButton('left', 'doubleClick');
1233
+ }),
1234
+ (0, device_namespaceObject.defineActionRightClick)(async ({ locate })=>{
1235
+ const element = this.requireLocate(locate, 'right click');
1236
+ await this.moveToElement(element, {
1237
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1238
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1239
+ });
1240
+ await this.backend.mouseButton('right', 'click');
1241
+ }),
1242
+ (0, device_namespaceObject.defineActionHover)(async ({ locate })=>{
1243
+ const element = this.requireLocate(locate, 'hover');
1244
+ await this.moveToElement(element, {
1245
+ steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1246
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE,
1247
+ settleDelayMs: device_MOUSE_MOVE_EFFECT_WAIT
1248
+ });
1249
+ }),
1250
+ (0, device_namespaceObject.defineActionInput)(async (param)=>{
1251
+ this.assertConnected();
1252
+ if (param.locate) {
1253
+ await this.moveToElement(param.locate, {
1254
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1255
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1256
+ });
1257
+ await this.backend.mouseButton('left', 'click');
1258
+ await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
1259
+ }
1260
+ if ('typeOnly' !== param.mode) {
1261
+ await this.clearInput();
1262
+ await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
1263
+ }
1264
+ if ('clear' === param.mode) return;
1265
+ if (param.value) await this.backend.typeText(param.value);
1266
+ }),
1267
+ (0, device_namespaceObject.defineActionClearInput)(async ({ locate })=>{
1268
+ this.assertConnected();
1269
+ if (locate) {
1270
+ await this.moveToElement(locate, {
1271
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1272
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1273
+ });
1274
+ await this.backend.mouseButton('left', 'click');
1275
+ await (0, utils_namespaceObject.sleep)(device_INPUT_FOCUS_DELAY);
1276
+ }
1277
+ await this.clearInput();
1278
+ await (0, utils_namespaceObject.sleep)(device_INPUT_CLEAR_DELAY);
1279
+ }),
1280
+ (0, device_namespaceObject.defineActionKeyboardPress)(async ({ locate, keyName })=>{
1281
+ this.assertConnected();
1282
+ if (locate) {
1283
+ await this.moveToElement(locate, {
1284
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1285
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1286
+ });
1287
+ await this.backend.mouseButton('left', 'click');
1288
+ }
1289
+ await this.backend.keyPress(keyName);
1290
+ }),
1291
+ (0, device_namespaceObject.defineActionScroll)(async (param)=>{
1292
+ this.assertConnected();
1293
+ const target = param.locate;
1294
+ if (target) await this.moveToElement(target, {
1295
+ steps: device_SMOOTH_MOVE_STEPS_MOUSE_MOVE,
1296
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_MOUSE_MOVE
1297
+ });
1298
+ if (param.scrollType && 'singleAction' !== param.scrollType) {
1299
+ const direction = this.edgeScrollDirection(param.scrollType);
1300
+ for(let i = 0; i < device_EDGE_SCROLL_STEPS; i++)await this.performWheel(direction, DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
1301
+ await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
1302
+ return;
1303
+ }
1304
+ await this.performWheel(param.direction || 'down', param.distance || DEFAULT_SCROLL_DISTANCE, target?.center[0], target?.center[1]);
1305
+ await (0, utils_namespaceObject.sleep)(device_SCROLL_COMPLETE_DELAY);
1306
+ }),
1307
+ (0, device_namespaceObject.defineActionDragAndDrop)(async ({ from, to })=>{
1308
+ this.assertConnected();
1309
+ const source = this.requireLocate(from, 'drag source');
1310
+ const target = this.requireLocate(to, 'drag target');
1311
+ await this.moveToElement(source, {
1312
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1313
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1314
+ });
1315
+ await this.backend.mouseButton('left', 'down');
1316
+ await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
1317
+ await this.moveToElement(target, {
1318
+ steps: SMOOTH_MOVE_STEPS_DRAG,
1319
+ stepDelayMs: SMOOTH_MOVE_DELAY_DRAG
1320
+ });
1321
+ await (0, utils_namespaceObject.sleep)(DRAG_HOLD_DURATION);
1322
+ await this.backend.mouseButton('left', 'up');
1323
+ }),
1324
+ (0, device_namespaceObject.defineAction)({
1325
+ name: 'MiddleClick',
1326
+ description: 'Middle click the element',
1327
+ sample: {
1328
+ locate: {
1329
+ prompt: 'the browser tab close target'
1330
+ }
1331
+ },
1332
+ paramSchema: device_namespaceObject.actionTapParamSchema,
1333
+ call: async ({ locate })=>{
1334
+ const element = this.requireLocate(locate, 'middle click');
1335
+ await this.moveToElement(element, {
1336
+ steps: device_SMOOTH_MOVE_STEPS_TAP,
1337
+ stepDelayMs: device_SMOOTH_MOVE_DELAY_TAP
1338
+ });
1339
+ await this.backend.mouseButton('middle', 'click');
1340
+ }
1341
+ }),
1342
+ (0, device_namespaceObject.defineAction)({
1343
+ name: 'ListDisplays',
1344
+ description: 'List all available displays/monitors',
1345
+ call: async ()=>{
1346
+ this.assertConnected();
1347
+ const size = await this.size();
1348
+ return [
1349
+ {
1350
+ id: this.connectionInfo?.sessionId || this.options.host,
1351
+ name: `RDP ${this.connectionInfo?.server || this.options.host} (${size.width}x${size.height})`,
1352
+ primary: true
1353
+ }
1354
+ ];
1355
+ }
1356
+ })
1357
+ ];
1358
+ return [
1359
+ ...defaultActions,
1360
+ ...this.options.customActions || []
1361
+ ];
1362
+ }
1363
+ assertConnected() {
1364
+ this.throwIfDestroyed();
1365
+ if (!this.connectionInfo) throw new Error('RDPDevice is not connected');
1366
+ }
1367
+ throwIfDestroyed() {
1368
+ if (this.destroyed) throw new Error('RDPDevice has been destroyed');
1369
+ }
1370
+ requireLocate(locate, actionName) {
1371
+ if (!locate) throw new Error(`Missing target element for ${actionName}`);
1372
+ return locate;
1373
+ }
1374
+ async moveToElement(element, options) {
1375
+ this.assertConnected();
1376
+ const targetX = Math.round(element.center[0]);
1377
+ const targetY = Math.round(element.center[1]);
1378
+ await this.movePointer(targetX, targetY, options);
1379
+ }
1380
+ async clearInput() {
1381
+ if (this.backend.clearInput) return void await this.backend.clearInput();
1382
+ await this.backend.keyPress('Control+A');
1383
+ await this.backend.keyPress('Backspace');
1384
+ }
1385
+ edgeScrollDirection(scrollType) {
1386
+ switch(scrollType){
1387
+ case 'scrollToTop':
1388
+ return 'up';
1389
+ case 'scrollToBottom':
1390
+ return 'down';
1391
+ case 'scrollToLeft':
1392
+ return 'left';
1393
+ case 'scrollToRight':
1394
+ return 'right';
1395
+ case 'singleAction':
1396
+ return 'down';
1397
+ default:
1398
+ throw new Error(`Unsupported scroll type: ${scrollType}`);
1399
+ }
1400
+ }
1401
+ async movePointer(targetX, targetY, options) {
1402
+ this.assertConnected();
1403
+ const start = this.cursorPosition || [
1404
+ targetX,
1405
+ targetY
1406
+ ];
1407
+ const steps = Math.max(1, options?.steps || 1);
1408
+ const stepDelayMs = options?.stepDelayMs || 0;
1409
+ for(let step = 1; step <= steps; step++){
1410
+ const x = Math.round(start[0] + (targetX - start[0]) * step / steps);
1411
+ const y = Math.round(start[1] + (targetY - start[1]) * step / steps);
1412
+ await this.backend.mouseMove(x, y);
1413
+ this.cursorPosition = [
1414
+ x,
1415
+ y
1416
+ ];
1417
+ if (stepDelayMs > 0 && step < steps) await (0, utils_namespaceObject.sleep)(stepDelayMs);
1418
+ }
1419
+ if (options?.settleDelayMs) await (0, utils_namespaceObject.sleep)(options.settleDelayMs);
1420
+ }
1421
+ async performWheel(direction, amount, x, y) {
1422
+ let remaining = Math.abs(amount);
1423
+ if (0 === remaining) remaining = DEFAULT_SCROLL_STEP_AMOUNT;
1424
+ while(remaining > 0){
1425
+ const chunk = Math.min(remaining, DEFAULT_SCROLL_STEP_AMOUNT);
1426
+ await this.backend.wheel(direction, chunk, x, y);
1427
+ remaining -= chunk;
1428
+ if (remaining > 0) await (0, utils_namespaceObject.sleep)(device_SCROLL_STEP_DELAY);
1429
+ }
1430
+ }
1431
+ constructor(options){
1432
+ device_define_property(this, "interfaceType", 'rdp');
1433
+ device_define_property(this, "options", void 0);
1434
+ device_define_property(this, "backend", void 0);
1435
+ device_define_property(this, "connectionInfo", void 0);
1436
+ device_define_property(this, "destroyed", false);
1437
+ device_define_property(this, "cursorPosition", void 0);
1438
+ device_define_property(this, "uri", void 0);
1439
+ this.options = {
1440
+ port: 3389,
1441
+ securityProtocol: 'auto',
1442
+ ignoreCertificate: false,
1443
+ ...options
1444
+ };
1445
+ this.backend = options.backend || createDefaultRDPBackendClient();
1446
+ }
1447
+ }
858
1448
  class ComputerAgent extends agent_namespaceObject.Agent {
859
1449
  }
860
1450
  function createLocalComputerDevice(opts) {
@@ -866,12 +1456,33 @@ function createLocalComputerDevice(opts) {
866
1456
  xvfbResolution: opts?.xvfbResolution
867
1457
  });
868
1458
  }
1459
+ function createRDPComputerDevice(opts) {
1460
+ return new RDPDevice({
1461
+ host: opts.host,
1462
+ port: opts.port,
1463
+ username: opts.username,
1464
+ password: opts.password,
1465
+ domain: opts.domain,
1466
+ adminSession: opts.adminSession,
1467
+ ignoreCertificate: opts.ignoreCertificate,
1468
+ securityProtocol: opts.securityProtocol,
1469
+ desktopWidth: opts.desktopWidth,
1470
+ desktopHeight: opts.desktopHeight,
1471
+ backend: opts.backend,
1472
+ customActions: opts.customActions
1473
+ });
1474
+ }
869
1475
  async function agentForComputer(opts) {
870
1476
  const device = createLocalComputerDevice(opts);
871
1477
  await device.connect();
872
1478
  return new ComputerAgent(device, opts);
873
1479
  }
874
1480
  const agentFromComputer = agentForComputer;
1481
+ async function agentForRDPComputer(opts) {
1482
+ const device = createRDPComputerDevice(opts);
1483
+ await device.connect();
1484
+ return new ComputerAgent(device, opts);
1485
+ }
875
1486
  const base_tools_namespaceObject = require("@midscene/shared/mcp/base-tools");
876
1487
  function mcp_tools_define_property(obj, key, value) {
877
1488
  if (key in obj) Object.defineProperty(obj, key, {
@@ -884,10 +1495,61 @@ function mcp_tools_define_property(obj, key, value) {
884
1495
  return obj;
885
1496
  }
886
1497
  const mcp_tools_debug = (0, logger_namespaceObject.getDebug)('mcp:computer-tools');
1498
+ const RDP_SECURITY_PROTOCOLS = [
1499
+ 'auto',
1500
+ 'tls',
1501
+ 'nla',
1502
+ 'rdp'
1503
+ ];
887
1504
  const computerInitArgShape = {
888
- displayId: core_namespaceObject.z.string().optional().describe('Display ID (from computer_list_displays)'),
889
- headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux only)')
1505
+ displayId: core_namespaceObject.z.string().optional().describe('Display ID for local mode (from computer_list_displays). Ignored when host is set.'),
1506
+ headless: core_namespaceObject.z.boolean().optional().describe('Start virtual display via Xvfb (Linux local mode only). Ignored when host is set.'),
1507
+ host: core_namespaceObject.z.string().optional().describe('RDP host (FQDN or IP). Set this to switch into RDP mode.'),
1508
+ port: core_namespaceObject.z.number().optional().describe('RDP port (default 3389). Requires host.'),
1509
+ username: core_namespaceObject.z.string().optional().describe('RDP username. Requires host.'),
1510
+ password: core_namespaceObject.z.string().optional().describe('RDP password. Requires host. Prefer setting via environment or a secrets manager.'),
1511
+ domain: core_namespaceObject.z.string().optional().describe('RDP domain. Requires host.'),
1512
+ adminSession: core_namespaceObject.z.boolean().optional().describe('Attach to the RDP admin/console session. Requires host.'),
1513
+ ignoreCertificate: core_namespaceObject.z.boolean().optional().describe('Skip TLS certificate validation. Requires host.'),
1514
+ securityProtocol: core_namespaceObject.z["enum"](RDP_SECURITY_PROTOCOLS).optional().describe('RDP security protocol negotiation (default auto). Requires host.'),
1515
+ desktopWidth: core_namespaceObject.z.number().optional().describe('Remote desktop width in pixels. Requires host.'),
1516
+ desktopHeight: core_namespaceObject.z.number().optional().describe('Remote desktop height in pixels. Requires host.')
890
1517
  };
1518
+ function adaptComputerInitArgs(extracted) {
1519
+ if (!extracted || 0 === Object.keys(extracted).length) return;
1520
+ if (extracted.host) {
1521
+ const { displayId: _d, headless: _h, ...rdpFields } = extracted;
1522
+ return {
1523
+ mode: 'rdp',
1524
+ ...rdpFields,
1525
+ host: extracted.host
1526
+ };
1527
+ }
1528
+ return {
1529
+ mode: 'local',
1530
+ displayId: extracted.displayId,
1531
+ headless: extracted.headless
1532
+ };
1533
+ }
1534
+ function shouldRetargetAgent(opts) {
1535
+ if (!opts) return false;
1536
+ if ('rdp' === opts.mode) return true;
1537
+ return void 0 !== opts.displayId || void 0 !== opts.headless;
1538
+ }
1539
+ function describeConnectTarget(opts) {
1540
+ if (opts?.mode === 'rdp') {
1541
+ const portSuffix = opts.port ? `:${opts.port}` : '';
1542
+ const userSuffix = opts.username ? ` as ${opts.username}` : '';
1543
+ return ` via RDP (${opts.host}${portSuffix}${userSuffix})`;
1544
+ }
1545
+ if (opts?.mode === 'local' && opts.displayId) return ` (Display: ${opts.displayId})`;
1546
+ return ' (Primary display)';
1547
+ }
1548
+ function getCliReportSessionTarget(opts) {
1549
+ if (opts?.mode === 'rdp') return `rdp:${opts.host}`;
1550
+ if (opts?.mode === 'local' && opts.displayId) return opts.displayId;
1551
+ return 'primary';
1552
+ }
891
1553
  class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
892
1554
  getCliReportSessionName() {
893
1555
  return 'midscene-computer';
@@ -896,9 +1558,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
896
1558
  return new ComputerDevice({});
897
1559
  }
898
1560
  async ensureAgent(opts) {
899
- const displayId = opts?.displayId;
900
- const headless = opts?.headless;
901
- if (this.agent && (void 0 !== displayId || void 0 !== headless)) {
1561
+ if (this.agent && shouldRetargetAgent(opts)) {
902
1562
  try {
903
1563
  await this.agent.destroy?.();
904
1564
  } catch (error) {
@@ -907,8 +1567,20 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
907
1567
  this.agent = void 0;
908
1568
  }
909
1569
  if (this.agent) return this.agent;
910
- mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
911
1570
  const reportOptions = this.readCliReportAgentOptions();
1571
+ if (opts?.mode === 'rdp') {
1572
+ mcp_tools_debug('Creating RDP Computer agent for host:', opts.host);
1573
+ const { mode: _mode, ...rdpFields } = opts;
1574
+ const agent = await agentForRDPComputer({
1575
+ ...rdpFields,
1576
+ ...reportOptions ?? {}
1577
+ });
1578
+ this.agent = agent;
1579
+ return agent;
1580
+ }
1581
+ const displayId = opts?.mode === 'local' ? opts.displayId : void 0;
1582
+ const headless = opts?.mode === 'local' ? opts.headless : void 0;
1583
+ mcp_tools_debug('Creating Computer agent with displayId:', displayId || 'primary');
912
1584
  const agentOpts = {
913
1585
  ...displayId ? {
914
1586
  displayId
@@ -926,12 +1598,12 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
926
1598
  return [
927
1599
  {
928
1600
  name: 'computer_connect',
929
- description: 'Connect to computer desktop. Provide displayId to connect to a specific display (use computer_list_displays to get available IDs). If not provided, uses the primary display.',
1601
+ description: "Connect to a computer desktop. Default (local) mode controls the local machine; pass displayId to target a specific local display (see computer_list_displays). Pass host to switch to RDP mode and connect to a remote Windows desktop via the RDP helper binary. RDP-related options (port/username/password/domain/securityProtocol/ignoreCertificate/adminSession/desktopWidth/desktopHeight) only take effect when host is set.",
930
1602
  schema: this.getAgentInitArgSchema(),
931
1603
  cli: this.getAgentInitArgCliMetadata(),
932
1604
  handler: async (args)=>{
933
1605
  const initArgs = this.extractAgentInitParam(args);
934
- const reportSession = this.createNewCliReportSession(initArgs?.displayId ?? 'primary');
1606
+ const reportSession = this.createNewCliReportSession(getCliReportSessionTarget(initArgs));
935
1607
  this.commitCliReportSession(reportSession);
936
1608
  if (this.agent) {
937
1609
  try {
@@ -947,7 +1619,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
947
1619
  content: [
948
1620
  {
949
1621
  type: 'text',
950
- text: `Connected to computer${initArgs?.displayId ? ` (Display: ${initArgs.displayId})` : ' (Primary display)'}`
1622
+ text: `Connected to computer${describeConnectTarget(initArgs)}`
951
1623
  },
952
1624
  ...this.buildScreenshotContent(screenshot)
953
1625
  ]
@@ -985,7 +1657,7 @@ class ComputerMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools
985
1657
  cli: {
986
1658
  preferBareKeys: true
987
1659
  },
988
- adapt: (extracted)=>extracted
1660
+ adapt: (extracted)=>adaptComputerInitArgs(extracted)
989
1661
  });
990
1662
  }
991
1663
  }
@@ -996,7 +1668,7 @@ class ComputerMCPServer extends mcp_namespaceObject.BaseMCPServer {
996
1668
  constructor(toolsManager){
997
1669
  super({
998
1670
  name: '@midscene/computer-mcp',
999
- version: "1.8.0",
1671
+ version: "1.8.1-beta-20260513084557.0",
1000
1672
  description: 'Control the computer desktop using natural language commands'
1001
1673
  }, toolsManager);
1002
1674
  }