@aiscene/android 1.6.7 → 1.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/es/cli.mjs +125 -95
- package/dist/es/index.mjs +124 -94
- package/dist/es/mcp-server.mjs +125 -95
- package/dist/lib/cli.js +123 -93
- package/dist/lib/index.js +122 -92
- package/dist/lib/mcp-server.js +123 -93
- package/dist/types/index.d.ts +40 -31
- package/dist/types/mcp-server.d.ts +40 -31
- package/package.json +24 -21
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -6,11 +6,12 @@ import { BaseMCPServer, BaseMidsceneTools, createMCPServerLauncher } from "@mids
|
|
|
6
6
|
import { Agent } from "@midscene/core/agent";
|
|
7
7
|
import { mergeAndNormalizeAppNameMapping, normalizeForComparison, repeat } from "@midscene/shared/utils";
|
|
8
8
|
import node_assert from "node:assert";
|
|
9
|
+
import { execFile } from "node:child_process";
|
|
9
10
|
import { getMidsceneLocationSchema, z } from "@midscene/core";
|
|
10
|
-
import { defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionScroll, defineActionTap } from "@midscene/core/device";
|
|
11
|
+
import { defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionPinch, defineActionScroll, defineActionSwipe, defineActionTap, normalizeMobileSwipeParam, normalizePinchParam } from "@midscene/core/device";
|
|
11
12
|
import { getTmpFile, sleep } from "@midscene/core/utils";
|
|
12
13
|
import { MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, globalConfigManager } from "@midscene/shared/env";
|
|
13
|
-
import { createImgBase64ByFormat,
|
|
14
|
+
import { createImgBase64ByFormat, isValidImageBuffer } from "@midscene/shared/img";
|
|
14
15
|
import { ADB } from "appium-adb";
|
|
15
16
|
var __webpack_modules__ = {
|
|
16
17
|
"./src/scrcpy-manager.ts" (__unused_rspack_module, __webpack_exports__, __webpack_require__) {
|
|
@@ -674,6 +675,9 @@ const defaultNormalScrollDuration = 1000;
|
|
|
674
675
|
const IME_STRATEGY_ALWAYS_YADB = 'always-yadb';
|
|
675
676
|
const IME_STRATEGY_YADB_FOR_NON_ASCII = 'yadb-for-non-ascii';
|
|
676
677
|
const debugDevice = (0, logger_.getDebug)('android:device');
|
|
678
|
+
function escapeForShell(text) {
|
|
679
|
+
return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
|
|
680
|
+
}
|
|
677
681
|
class AndroidDevice {
|
|
678
682
|
actionSpace() {
|
|
679
683
|
const defaultActions = [
|
|
@@ -701,6 +705,12 @@ class AndroidDevice {
|
|
|
701
705
|
]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - clear the field without inputting new text.')),
|
|
702
706
|
locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
|
|
703
707
|
}),
|
|
708
|
+
sample: {
|
|
709
|
+
value: 'test@example.com',
|
|
710
|
+
locate: {
|
|
711
|
+
prompt: 'the email input field'
|
|
712
|
+
}
|
|
713
|
+
},
|
|
704
714
|
call: async (param)=>{
|
|
705
715
|
const element = param.locate;
|
|
706
716
|
if ('typeOnly' !== param.mode) await this.clearInput(element);
|
|
@@ -746,9 +756,21 @@ class AndroidDevice {
|
|
|
746
756
|
y: to.center[1]
|
|
747
757
|
});
|
|
748
758
|
}),
|
|
759
|
+
defineActionSwipe(async (param)=>{
|
|
760
|
+
const { startPoint, endPoint, duration, repeatCount } = normalizeMobileSwipeParam(param, await this.size());
|
|
761
|
+
for(let i = 0; i < repeatCount; i++)await this.mouseDrag(startPoint, endPoint, duration);
|
|
762
|
+
}),
|
|
749
763
|
defineActionKeyboardPress(async (param)=>{
|
|
750
764
|
await this.keyboardPress(param.keyName);
|
|
751
765
|
}),
|
|
766
|
+
defineActionCursorMove(async (param)=>{
|
|
767
|
+
const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
|
|
768
|
+
const times = param.times ?? 1;
|
|
769
|
+
for(let i = 0; i < times; i++){
|
|
770
|
+
await this.keyboardPress(arrowKey);
|
|
771
|
+
await sleep(100);
|
|
772
|
+
}
|
|
773
|
+
}),
|
|
752
774
|
defineAction({
|
|
753
775
|
name: 'LongPress',
|
|
754
776
|
description: 'Trigger a long press on the screen at specified element',
|
|
@@ -756,6 +778,11 @@ class AndroidDevice {
|
|
|
756
778
|
duration: z.number().optional().describe('The duration of the long press in milliseconds'),
|
|
757
779
|
locate: getMidsceneLocationSchema().describe('The element to be long pressed')
|
|
758
780
|
}),
|
|
781
|
+
sample: {
|
|
782
|
+
locate: {
|
|
783
|
+
prompt: 'the message bubble'
|
|
784
|
+
}
|
|
785
|
+
},
|
|
759
786
|
call: async (param)=>{
|
|
760
787
|
const element = param.locate;
|
|
761
788
|
if (!element) throw new Error('LongPress requires an element to be located');
|
|
@@ -775,6 +802,12 @@ class AndroidDevice {
|
|
|
775
802
|
duration: z.number().optional().describe('The duration of the pull (in milliseconds)'),
|
|
776
803
|
locate: getMidsceneLocationSchema().optional().describe('The element to start the pull from (optional)')
|
|
777
804
|
}),
|
|
805
|
+
sample: {
|
|
806
|
+
direction: 'down',
|
|
807
|
+
locate: {
|
|
808
|
+
prompt: 'the center of the content list area'
|
|
809
|
+
}
|
|
810
|
+
},
|
|
778
811
|
call: async (param)=>{
|
|
779
812
|
const element = param.locate;
|
|
780
813
|
const startPoint = element ? {
|
|
@@ -787,6 +820,16 @@ class AndroidDevice {
|
|
|
787
820
|
else throw new Error(`Unknown pull direction: ${param.direction}`);
|
|
788
821
|
}
|
|
789
822
|
}),
|
|
823
|
+
defineActionPinch(async (param)=>{
|
|
824
|
+
const { centerX, centerY, startDistance, endDistance, duration } = normalizePinchParam(param, await this.size());
|
|
825
|
+
const { x: adjCenterX, y: adjCenterY } = await this.adjustCoordinates(centerX, centerY);
|
|
826
|
+
const ratio = 0 !== adjCenterX && 0 !== centerX ? adjCenterX / centerX : 1;
|
|
827
|
+
const adjStartDist = Math.round(startDistance * ratio);
|
|
828
|
+
const adjEndDist = Math.round(endDistance * ratio);
|
|
829
|
+
await this.ensureYadb();
|
|
830
|
+
const adb = await this.getAdb();
|
|
831
|
+
await adb.shell(`app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -pinch ${adjCenterX} ${adjCenterY} ${adjStartDist} ${adjEndDist} ${duration}`);
|
|
832
|
+
}),
|
|
790
833
|
defineActionClearInput(async (param)=>{
|
|
791
834
|
await this.clearInput(param.locate);
|
|
792
835
|
})
|
|
@@ -931,66 +974,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
931
974
|
async execYadb(keyboardContent) {
|
|
932
975
|
await this.ensureYadb();
|
|
933
976
|
const adb = await this.getAdb();
|
|
934
|
-
|
|
935
|
-
const command = `app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "${keyboardContent}"`;
|
|
936
|
-
debugDevice(`Executing YADB input: "${keyboardContent}"`);
|
|
937
|
-
const inputPromise = adb.shell(command);
|
|
938
|
-
const timeoutPromise = new Promise((_, reject)=>setTimeout(()=>reject(new Error('YADB timeout')), 1000));
|
|
939
|
-
await Promise.race([
|
|
940
|
-
inputPromise,
|
|
941
|
-
timeoutPromise
|
|
942
|
-
]);
|
|
943
|
-
debugDevice(`YADB input completed: "${keyboardContent}"`);
|
|
944
|
-
} catch (error) {
|
|
945
|
-
const isAccessibilityConflict = error?.cause?.stderr?.includes('UiAutomationService') || error?.cause?.stderr?.includes('already registered') || error?.message?.includes('UiAutomationService');
|
|
946
|
-
if (isAccessibilityConflict) {
|
|
947
|
-
debugDevice("YADB failed due to AccessibilityService conflict (likely Appium running), falling back to clipboard method");
|
|
948
|
-
await this.inputViaClipboard(keyboardContent);
|
|
949
|
-
} else 'YADB timeout' === error.message ? debugDevice(`YADB timed out after 2s, assuming input succeeded: "${keyboardContent}"`) : debugDevice(`YADB execution may have completed despite error: ${error}`);
|
|
950
|
-
}
|
|
951
|
-
}
|
|
952
|
-
async inputViaClipboard(text) {
|
|
953
|
-
const adb = await this.getAdb();
|
|
954
|
-
try {
|
|
955
|
-
debugDevice(`Inputting via clipboard: "${text}"`);
|
|
956
|
-
const escapedText = text.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
957
|
-
const setClipboardCmd = `
|
|
958
|
-
content insert --uri content://settings/system --bind name:s:clipboard_text --bind value:s:"${escapedText}"
|
|
959
|
-
`;
|
|
960
|
-
await adb.shell(setClipboardCmd);
|
|
961
|
-
await sleep(100);
|
|
962
|
-
await adb.shell('input keyevent KEYCODE_PASTE');
|
|
963
|
-
await sleep(100);
|
|
964
|
-
debugDevice(`Clipboard input completed via content provider: "${text}"`);
|
|
965
|
-
} catch (error1) {
|
|
966
|
-
debugDevice(`Content provider clipboard failed, trying clipper app: ${error1}`);
|
|
967
|
-
try {
|
|
968
|
-
const base64Text = Buffer.from(text, 'utf-8').toString('base64');
|
|
969
|
-
await adb.shell(`am broadcast -a clipper.set -e text "${base64Text}"`);
|
|
970
|
-
await sleep(100);
|
|
971
|
-
await adb.shell('input keyevent KEYCODE_PASTE');
|
|
972
|
-
await sleep(100);
|
|
973
|
-
debugDevice(`Clipboard input completed via clipper: "${text}"`);
|
|
974
|
-
} catch (error2) {
|
|
975
|
-
debugDevice(`All clipboard methods failed: ${error2}`);
|
|
976
|
-
const isPureAscii = /^[\x00-\x7F]*$/.test(text);
|
|
977
|
-
if (isPureAscii) {
|
|
978
|
-
debugDevice(`Using ADB inputText for ASCII text: "${text}"`);
|
|
979
|
-
await adb.inputText(text);
|
|
980
|
-
} else await this.inputCharByChar(text);
|
|
981
|
-
}
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
async inputCharByChar(text) {
|
|
985
|
-
const adb = await this.getAdb();
|
|
986
|
-
debugDevice(`Inputting character by character (slow method): "${text}"`);
|
|
987
|
-
const chars = Array.from(text);
|
|
988
|
-
for (const char of chars){
|
|
989
|
-
if (' ' === char) await adb.shell('input keyevent KEYCODE_SPACE');
|
|
990
|
-
else await adb.shell(`input text "${char}"`);
|
|
991
|
-
await sleep(50);
|
|
992
|
-
}
|
|
993
|
-
debugDevice("Character-by-character input completed");
|
|
977
|
+
await adb.shell(`app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard '${keyboardContent}'`);
|
|
994
978
|
}
|
|
995
979
|
async getElementsInfo() {
|
|
996
980
|
return [];
|
|
@@ -1003,6 +987,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1003
987
|
}
|
|
1004
988
|
async getScreenSize() {
|
|
1005
989
|
const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
|
|
990
|
+
debugDevice(`getScreenSize: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedSize=${!!this.cachedScreenSize}`);
|
|
1006
991
|
if (shouldCache && this.cachedScreenSize) return this.cachedScreenSize;
|
|
1007
992
|
const adb = await this.getAdb();
|
|
1008
993
|
if ('number' == typeof this.options?.displayId) try {
|
|
@@ -1136,6 +1121,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1136
1121
|
}
|
|
1137
1122
|
async getDisplayOrientation() {
|
|
1138
1123
|
const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
|
|
1124
|
+
debugDevice(`getDisplayOrientation: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedOrientation=${null !== this.cachedOrientation}`);
|
|
1139
1125
|
if (shouldCache && null !== this.cachedOrientation) return this.cachedOrientation;
|
|
1140
1126
|
const adb = await this.getAdb();
|
|
1141
1127
|
let orientation = 0;
|
|
@@ -1161,6 +1147,15 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1161
1147
|
if (shouldCache) this.cachedOrientation = orientation;
|
|
1162
1148
|
return orientation;
|
|
1163
1149
|
}
|
|
1150
|
+
async getOrientedPhysicalSize() {
|
|
1151
|
+
const info = await this.getDevicePhysicalInfo();
|
|
1152
|
+
const isLandscape = 1 === info.orientation || 3 === info.orientation;
|
|
1153
|
+
const shouldSwap = true !== info.isCurrentOrientation && isLandscape;
|
|
1154
|
+
return {
|
|
1155
|
+
width: shouldSwap ? info.physicalHeight : info.physicalWidth,
|
|
1156
|
+
height: shouldSwap ? info.physicalWidth : info.physicalHeight
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
1164
1159
|
async size() {
|
|
1165
1160
|
const deviceInfo = await this.getDevicePhysicalInfo();
|
|
1166
1161
|
const adapter = this.getScrcpyAdapter();
|
|
@@ -1187,7 +1182,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1187
1182
|
height: logicalHeight
|
|
1188
1183
|
};
|
|
1189
1184
|
}
|
|
1190
|
-
async cacheFeatureForPoint(center
|
|
1185
|
+
async cacheFeatureForPoint(center) {
|
|
1191
1186
|
const { width, height } = await this.size();
|
|
1192
1187
|
debugDevice('cacheFeatureForPoint: center=[%s,%s], screen=[%s,%s]', center[0], center[1], width, height);
|
|
1193
1188
|
return {
|
|
@@ -1267,14 +1262,23 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1267
1262
|
const androidScreenshotPath = `/data/local/tmp/ms_${screenshotId}.png`;
|
|
1268
1263
|
const useShellScreencap = 'number' == typeof this.options?.displayId;
|
|
1269
1264
|
try {
|
|
1270
|
-
if (useShellScreencap
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1265
|
+
if (!useShellScreencap && this.takeScreenshotFailCount < AndroidDevice.TAKE_SCREENSHOT_FAIL_THRESHOLD) {
|
|
1266
|
+
debugDevice('Taking screenshot via adb.takeScreenshot');
|
|
1267
|
+
screenshotBuffer = await adb.takeScreenshot(null);
|
|
1268
|
+
debugDevice('adb.takeScreenshot completed');
|
|
1269
|
+
if (!screenshotBuffer) {
|
|
1270
|
+
this.takeScreenshotFailCount++;
|
|
1271
|
+
throw new Error('Failed to capture screenshot: screenshotBuffer is null');
|
|
1272
|
+
}
|
|
1273
|
+
if (!isValidImageBuffer(screenshotBuffer)) {
|
|
1274
|
+
debugDevice('Invalid image buffer detected: not a valid image format');
|
|
1275
|
+
this.takeScreenshotFailCount++;
|
|
1276
|
+
throw new Error('Screenshot buffer has invalid format: could not find valid image signature');
|
|
1277
|
+
}
|
|
1278
|
+
this.takeScreenshotFailCount = 0;
|
|
1279
|
+
} else {
|
|
1280
|
+
if (this.takeScreenshotFailCount >= AndroidDevice.TAKE_SCREENSHOT_FAIL_THRESHOLD) debugDevice('Skipping takeScreenshot (failed %d consecutive times), using shell screencap directly', this.takeScreenshotFailCount);
|
|
1281
|
+
throw new Error('Using shell screencap directly');
|
|
1278
1282
|
}
|
|
1279
1283
|
const validScreenshotBufferSize = this.options?.minScreenshotBufferSize ?? 10240;
|
|
1280
1284
|
if (validScreenshotBufferSize > 0 && screenshotBuffer.length < validScreenshotBufferSize) {
|
|
@@ -1303,12 +1307,21 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1303
1307
|
screenshotBuffer = await external_node_fs_["default"].promises.readFile(screenshotPath);
|
|
1304
1308
|
const validScreenshotBufferSize = this.options?.minScreenshotBufferSize ?? 10240;
|
|
1305
1309
|
if (!screenshotBuffer || validScreenshotBufferSize > 0 && screenshotBuffer.length < validScreenshotBufferSize) throw new Error(`Fallback screenshot validation failed: buffer size ${screenshotBuffer?.length || 0} bytes (minimum: ${validScreenshotBufferSize})`);
|
|
1306
|
-
if (!
|
|
1310
|
+
if (!isValidImageBuffer(screenshotBuffer)) throw new Error('Fallback screenshot buffer has invalid PNG format');
|
|
1307
1311
|
debugDevice(`Fallback screenshot validated successfully: ${screenshotBuffer.length} bytes`);
|
|
1308
1312
|
} finally{
|
|
1309
|
-
|
|
1310
|
-
|
|
1313
|
+
const adbPath = adb.executable?.path ?? 'adb';
|
|
1314
|
+
const child = execFile(adbPath, [
|
|
1315
|
+
'-s',
|
|
1316
|
+
this.deviceId,
|
|
1317
|
+
'shell',
|
|
1318
|
+
`rm ${androidScreenshotPath}`
|
|
1319
|
+
], {
|
|
1320
|
+
timeout: 3000
|
|
1321
|
+
}, (err)=>{
|
|
1322
|
+
if (err) debugDevice('Failed to delete remote screenshot: %s', err.message);
|
|
1311
1323
|
});
|
|
1324
|
+
child.unref();
|
|
1312
1325
|
}
|
|
1313
1326
|
}
|
|
1314
1327
|
if (!screenshotBuffer) throw new Error('Failed to capture screenshot: all methods failed');
|
|
@@ -1471,24 +1484,32 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1471
1484
|
if (!this.yadbPushed) {
|
|
1472
1485
|
const adb = await this.getAdb();
|
|
1473
1486
|
const androidPkgJson = (0, external_node_module_.createRequire)(import.meta.url).resolve('@aiscene/android/package.json');
|
|
1474
|
-
const
|
|
1475
|
-
await adb.push(
|
|
1487
|
+
const yadbBin = external_node_path_["default"].join(external_node_path_["default"].dirname(androidPkgJson), 'bin', 'yadb');
|
|
1488
|
+
await adb.push(yadbBin, '/data/local/tmp');
|
|
1476
1489
|
this.yadbPushed = true;
|
|
1477
1490
|
}
|
|
1478
1491
|
}
|
|
1479
1492
|
shouldUseYadbForText(text) {
|
|
1480
1493
|
const hasNonAscii = /[\x80-\uFFFF]/.test(text);
|
|
1481
1494
|
const hasFormatSpecifiers = /%[a-zA-Z]/.test(text);
|
|
1482
|
-
|
|
1495
|
+
const hasShellSpecialChars = /[\\`$]/.test(text);
|
|
1496
|
+
const hasBothQuotes = text.includes('"') && text.includes("'");
|
|
1497
|
+
return hasNonAscii || hasFormatSpecifiers || hasShellSpecialChars || hasBothQuotes;
|
|
1483
1498
|
}
|
|
1484
1499
|
async keyboardType(text, options) {
|
|
1485
1500
|
if (!text) return;
|
|
1486
1501
|
const adb = await this.getAdb();
|
|
1487
|
-
const shouldUseYadb = this.shouldUseYadbForText(text);
|
|
1488
1502
|
const IME_STRATEGY = (this.options?.imeStrategy || globalConfigManager.getEnvConfigValue(MIDSCENE_ANDROID_IME_STRATEGY)) ?? IME_STRATEGY_YADB_FOR_NON_ASCII;
|
|
1489
1503
|
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
1490
|
-
|
|
1491
|
-
|
|
1504
|
+
const useYadb = IME_STRATEGY === IME_STRATEGY_ALWAYS_YADB || IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII && this.shouldUseYadbForText(text);
|
|
1505
|
+
if (useYadb) await this.execYadb(escapeForShell(text));
|
|
1506
|
+
else {
|
|
1507
|
+
const segments = text.split('\n');
|
|
1508
|
+
for(let i = 0; i < segments.length; i++){
|
|
1509
|
+
if (segments[i].length > 0) await adb.inputText(segments[i]);
|
|
1510
|
+
if (i < segments.length - 1) await adb.keyevent(66);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1492
1513
|
if (true === shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
1493
1514
|
}
|
|
1494
1515
|
normalizeKeyName(key) {
|
|
@@ -1536,12 +1557,12 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1536
1557
|
}
|
|
1537
1558
|
async mouseClick(x, y) {
|
|
1538
1559
|
const adb = await this.getAdb();
|
|
1539
|
-
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
1560
|
+
const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
|
|
1540
1561
|
await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} 150`);
|
|
1541
1562
|
}
|
|
1542
1563
|
async mouseDoubleClick(x, y) {
|
|
1543
1564
|
const adb = await this.getAdb();
|
|
1544
|
-
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
1565
|
+
const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
|
|
1545
1566
|
const tapCommand = `input${this.getDisplayArg()} tap ${adjustedX} ${adjustedY}`;
|
|
1546
1567
|
await adb.shell(tapCommand);
|
|
1547
1568
|
await sleep(50);
|
|
@@ -1552,8 +1573,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1552
1573
|
}
|
|
1553
1574
|
async mouseDrag(from, to, duration) {
|
|
1554
1575
|
const adb = await this.getAdb();
|
|
1555
|
-
const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
|
|
1556
|
-
const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
|
|
1576
|
+
const { x: fromX, y: fromY } = await this.adjustCoordinates(from.x, from.y);
|
|
1577
|
+
const { x: toX, y: toY } = await this.adjustCoordinates(to.x, to.y);
|
|
1557
1578
|
const swipeDuration = duration ?? defaultNormalScrollDuration;
|
|
1558
1579
|
await adb.shell(`input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`);
|
|
1559
1580
|
}
|
|
@@ -1563,16 +1584,16 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1563
1584
|
const n = 4;
|
|
1564
1585
|
const startX = Math.round(deltaX < 0 ? width / n * (n - 1) : width / n);
|
|
1565
1586
|
const startY = Math.round(deltaY < 0 ? height / n * (n - 1) : height / n);
|
|
1566
|
-
const
|
|
1567
|
-
const
|
|
1568
|
-
const
|
|
1569
|
-
const
|
|
1587
|
+
const maxPositiveDeltaX = startX;
|
|
1588
|
+
const maxNegativeDeltaX = width - startX;
|
|
1589
|
+
const maxPositiveDeltaY = startY;
|
|
1590
|
+
const maxNegativeDeltaY = height - startY;
|
|
1570
1591
|
deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
|
|
1571
1592
|
deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
|
|
1572
1593
|
const endX = Math.round(startX - deltaX);
|
|
1573
1594
|
const endY = Math.round(startY - deltaY);
|
|
1574
|
-
const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(startX, startY);
|
|
1575
|
-
const { x: adjustedEndX, y: adjustedEndY } = this.adjustCoordinates(endX, endY);
|
|
1595
|
+
const { x: adjustedStartX, y: adjustedStartY } = await this.adjustCoordinates(startX, startY);
|
|
1596
|
+
const { x: adjustedEndX, y: adjustedEndY } = await this.adjustCoordinates(endX, endY);
|
|
1576
1597
|
const adb = await this.getAdb();
|
|
1577
1598
|
const swipeDuration = duration ?? defaultNormalScrollDuration;
|
|
1578
1599
|
await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`);
|
|
@@ -1583,6 +1604,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1583
1604
|
this.cachedPhysicalDisplayId = void 0;
|
|
1584
1605
|
this.cachedScreenSize = null;
|
|
1585
1606
|
this.cachedOrientation = null;
|
|
1607
|
+
this.scalingRatio = 1;
|
|
1586
1608
|
if (this.scrcpyAdapter) {
|
|
1587
1609
|
await this.scrcpyAdapter.disconnect();
|
|
1588
1610
|
this.scrcpyAdapter = null;
|
|
@@ -1622,7 +1644,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1622
1644
|
}
|
|
1623
1645
|
async longPress(x, y, duration = 2000) {
|
|
1624
1646
|
const adb = await this.getAdb();
|
|
1625
|
-
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
1647
|
+
const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
|
|
1626
1648
|
await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} ${duration}`);
|
|
1627
1649
|
}
|
|
1628
1650
|
async pullDown(startPoint, distance, duration = 800) {
|
|
@@ -1644,8 +1666,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1644
1666
|
}
|
|
1645
1667
|
async pullDrag(from, to, duration) {
|
|
1646
1668
|
const adb = await this.getAdb();
|
|
1647
|
-
const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
|
|
1648
|
-
const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
|
|
1669
|
+
const { x: fromX, y: fromY } = await this.adjustCoordinates(from.x, from.y);
|
|
1670
|
+
const { x: toX, y: toY } = await this.adjustCoordinates(to.x, to.y);
|
|
1649
1671
|
await adb.shell(`input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${duration}`);
|
|
1650
1672
|
}
|
|
1651
1673
|
async pullUp(startPoint, distance, duration = 600) {
|
|
@@ -1732,7 +1754,6 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1732
1754
|
device_define_property(this, "yadbPushed", false);
|
|
1733
1755
|
device_define_property(this, "devicePixelRatio", 1);
|
|
1734
1756
|
device_define_property(this, "devicePixelRatioInitialized", false);
|
|
1735
|
-
device_define_property(this, "scalingRatio", 1);
|
|
1736
1757
|
device_define_property(this, "adb", null);
|
|
1737
1758
|
device_define_property(this, "connectingAdb", null);
|
|
1738
1759
|
device_define_property(this, "destroyed", false);
|
|
@@ -1743,6 +1764,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1743
1764
|
device_define_property(this, "cachedPhysicalDisplayId", void 0);
|
|
1744
1765
|
device_define_property(this, "scrcpyAdapter", null);
|
|
1745
1766
|
device_define_property(this, "appNameMapping", {});
|
|
1767
|
+
device_define_property(this, "scalingRatio", 1);
|
|
1768
|
+
device_define_property(this, "takeScreenshotFailCount", 0);
|
|
1746
1769
|
device_define_property(this, "interfaceType", 'android');
|
|
1747
1770
|
device_define_property(this, "uri", void 0);
|
|
1748
1771
|
device_define_property(this, "options", void 0);
|
|
@@ -1752,6 +1775,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1752
1775
|
this.customActions = options?.customActions;
|
|
1753
1776
|
}
|
|
1754
1777
|
}
|
|
1778
|
+
device_define_property(AndroidDevice, "TAKE_SCREENSHOT_FAIL_THRESHOLD", 3);
|
|
1755
1779
|
const runAdbShellParamSchema = z.object({
|
|
1756
1780
|
command: z.string().describe('ADB shell command to execute')
|
|
1757
1781
|
});
|
|
@@ -1764,6 +1788,9 @@ const createPlatformActions = (device)=>({
|
|
|
1764
1788
|
description: 'Execute ADB shell command on Android device',
|
|
1765
1789
|
interfaceAlias: 'runAdbShell',
|
|
1766
1790
|
paramSchema: runAdbShellParamSchema,
|
|
1791
|
+
sample: {
|
|
1792
|
+
command: 'dumpsys window displays | grep -E "mCurrentFocus"'
|
|
1793
|
+
},
|
|
1767
1794
|
call: async (param)=>{
|
|
1768
1795
|
if (!param.command || '' === param.command.trim()) throw new Error('RunAdbShell requires a non-empty command parameter');
|
|
1769
1796
|
const adb = await device.getAdb();
|
|
@@ -1775,6 +1802,9 @@ const createPlatformActions = (device)=>({
|
|
|
1775
1802
|
description: 'Launch an Android app or URL',
|
|
1776
1803
|
interfaceAlias: 'launch',
|
|
1777
1804
|
paramSchema: launchParamSchema,
|
|
1805
|
+
sample: {
|
|
1806
|
+
uri: 'com.example.app'
|
|
1807
|
+
},
|
|
1778
1808
|
call: async (param)=>{
|
|
1779
1809
|
if (!param.uri || '' === param.uri.trim()) throw new Error('Launch requires a non-empty uri parameter');
|
|
1780
1810
|
await device.launch(param.uri);
|
|
@@ -1927,7 +1957,7 @@ class AndroidMCPServer extends BaseMCPServer {
|
|
|
1927
1957
|
constructor(toolsManager){
|
|
1928
1958
|
super({
|
|
1929
1959
|
name: '@midscene/android-mcp',
|
|
1930
|
-
version: "1.6.
|
|
1960
|
+
version: "1.6.9",
|
|
1931
1961
|
description: 'Control the Android device using natural language commands'
|
|
1932
1962
|
}, toolsManager);
|
|
1933
1963
|
}
|