@aiscene/android 1.6.6 → 1.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/es/cli.mjs +125 -90
- package/dist/es/index.mjs +124 -89
- package/dist/es/mcp-server.mjs +125 -90
- package/dist/lib/cli.js +123 -88
- package/dist/lib/index.js +122 -87
- package/dist/lib/mcp-server.js +123 -88
- package/dist/types/index.d.ts +40 -31
- package/dist/types/mcp-server.d.ts +40 -31
- package/package.json +24 -21
package/dist/es/mcp-server.mjs
CHANGED
|
@@ -6,11 +6,12 @@ import { BaseMCPServer, BaseMidsceneTools, createMCPServerLauncher } from "@mids
|
|
|
6
6
|
import { Agent } from "@midscene/core/agent";
|
|
7
7
|
import { mergeAndNormalizeAppNameMapping, normalizeForComparison, repeat } from "@midscene/shared/utils";
|
|
8
8
|
import node_assert from "node:assert";
|
|
9
|
+
import { execFile } from "node:child_process";
|
|
9
10
|
import { getMidsceneLocationSchema, z } from "@midscene/core";
|
|
10
|
-
import { defineAction, defineActionClearInput, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionScroll, defineActionTap } from "@midscene/core/device";
|
|
11
|
+
import { defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionPinch, defineActionScroll, defineActionSwipe, defineActionTap, normalizeMobileSwipeParam, normalizePinchParam } from "@midscene/core/device";
|
|
11
12
|
import { getTmpFile, sleep } from "@midscene/core/utils";
|
|
12
13
|
import { MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, globalConfigManager } from "@midscene/shared/env";
|
|
13
|
-
import { createImgBase64ByFormat,
|
|
14
|
+
import { createImgBase64ByFormat, isValidImageBuffer } from "@midscene/shared/img";
|
|
14
15
|
import { ADB } from "appium-adb";
|
|
15
16
|
var __webpack_modules__ = {
|
|
16
17
|
"./src/scrcpy-manager.ts" (__unused_rspack_module, __webpack_exports__, __webpack_require__) {
|
|
@@ -674,6 +675,9 @@ const defaultNormalScrollDuration = 1000;
|
|
|
674
675
|
const IME_STRATEGY_ALWAYS_YADB = 'always-yadb';
|
|
675
676
|
const IME_STRATEGY_YADB_FOR_NON_ASCII = 'yadb-for-non-ascii';
|
|
676
677
|
const debugDevice = (0, logger_.getDebug)('android:device');
|
|
678
|
+
function escapeForShell(text) {
|
|
679
|
+
return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
|
|
680
|
+
}
|
|
677
681
|
class AndroidDevice {
|
|
678
682
|
actionSpace() {
|
|
679
683
|
const defaultActions = [
|
|
@@ -701,6 +705,12 @@ class AndroidDevice {
|
|
|
701
705
|
]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - clear the field without inputting new text.')),
|
|
702
706
|
locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
|
|
703
707
|
}),
|
|
708
|
+
sample: {
|
|
709
|
+
value: 'test@example.com',
|
|
710
|
+
locate: {
|
|
711
|
+
prompt: 'the email input field'
|
|
712
|
+
}
|
|
713
|
+
},
|
|
704
714
|
call: async (param)=>{
|
|
705
715
|
const element = param.locate;
|
|
706
716
|
if ('typeOnly' !== param.mode) await this.clearInput(element);
|
|
@@ -746,9 +756,21 @@ class AndroidDevice {
|
|
|
746
756
|
y: to.center[1]
|
|
747
757
|
});
|
|
748
758
|
}),
|
|
759
|
+
defineActionSwipe(async (param)=>{
|
|
760
|
+
const { startPoint, endPoint, duration, repeatCount } = normalizeMobileSwipeParam(param, await this.size());
|
|
761
|
+
for(let i = 0; i < repeatCount; i++)await this.mouseDrag(startPoint, endPoint, duration);
|
|
762
|
+
}),
|
|
749
763
|
defineActionKeyboardPress(async (param)=>{
|
|
750
764
|
await this.keyboardPress(param.keyName);
|
|
751
765
|
}),
|
|
766
|
+
defineActionCursorMove(async (param)=>{
|
|
767
|
+
const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
|
|
768
|
+
const times = param.times ?? 1;
|
|
769
|
+
for(let i = 0; i < times; i++){
|
|
770
|
+
await this.keyboardPress(arrowKey);
|
|
771
|
+
await sleep(100);
|
|
772
|
+
}
|
|
773
|
+
}),
|
|
752
774
|
defineAction({
|
|
753
775
|
name: 'LongPress',
|
|
754
776
|
description: 'Trigger a long press on the screen at specified element',
|
|
@@ -756,6 +778,11 @@ class AndroidDevice {
|
|
|
756
778
|
duration: z.number().optional().describe('The duration of the long press in milliseconds'),
|
|
757
779
|
locate: getMidsceneLocationSchema().describe('The element to be long pressed')
|
|
758
780
|
}),
|
|
781
|
+
sample: {
|
|
782
|
+
locate: {
|
|
783
|
+
prompt: 'the message bubble'
|
|
784
|
+
}
|
|
785
|
+
},
|
|
759
786
|
call: async (param)=>{
|
|
760
787
|
const element = param.locate;
|
|
761
788
|
if (!element) throw new Error('LongPress requires an element to be located');
|
|
@@ -775,6 +802,12 @@ class AndroidDevice {
|
|
|
775
802
|
duration: z.number().optional().describe('The duration of the pull (in milliseconds)'),
|
|
776
803
|
locate: getMidsceneLocationSchema().optional().describe('The element to start the pull from (optional)')
|
|
777
804
|
}),
|
|
805
|
+
sample: {
|
|
806
|
+
direction: 'down',
|
|
807
|
+
locate: {
|
|
808
|
+
prompt: 'the center of the content list area'
|
|
809
|
+
}
|
|
810
|
+
},
|
|
778
811
|
call: async (param)=>{
|
|
779
812
|
const element = param.locate;
|
|
780
813
|
const startPoint = element ? {
|
|
@@ -787,6 +820,16 @@ class AndroidDevice {
|
|
|
787
820
|
else throw new Error(`Unknown pull direction: ${param.direction}`);
|
|
788
821
|
}
|
|
789
822
|
}),
|
|
823
|
+
defineActionPinch(async (param)=>{
|
|
824
|
+
const { centerX, centerY, startDistance, endDistance, duration } = normalizePinchParam(param, await this.size());
|
|
825
|
+
const { x: adjCenterX, y: adjCenterY } = await this.adjustCoordinates(centerX, centerY);
|
|
826
|
+
const ratio = 0 !== adjCenterX && 0 !== centerX ? adjCenterX / centerX : 1;
|
|
827
|
+
const adjStartDist = Math.round(startDistance * ratio);
|
|
828
|
+
const adjEndDist = Math.round(endDistance * ratio);
|
|
829
|
+
await this.ensureYadb();
|
|
830
|
+
const adb = await this.getAdb();
|
|
831
|
+
await adb.shell(`app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -pinch ${adjCenterX} ${adjCenterY} ${adjStartDist} ${adjEndDist} ${duration}`);
|
|
832
|
+
}),
|
|
790
833
|
defineActionClearInput(async (param)=>{
|
|
791
834
|
await this.clearInput(param.locate);
|
|
792
835
|
})
|
|
@@ -931,61 +974,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
931
974
|
async execYadb(keyboardContent) {
|
|
932
975
|
await this.ensureYadb();
|
|
933
976
|
const adb = await this.getAdb();
|
|
934
|
-
|
|
935
|
-
const command = `app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "${keyboardContent}"`;
|
|
936
|
-
debugDevice(`Executing YADB input: "${keyboardContent}"`);
|
|
937
|
-
await adb.shell(command);
|
|
938
|
-
debugDevice(`YADB input completed: "${keyboardContent}"`);
|
|
939
|
-
} catch (error) {
|
|
940
|
-
const isAccessibilityConflict = error?.cause?.stderr?.includes('UiAutomationService') || error?.cause?.stderr?.includes('already registered') || error?.message?.includes('UiAutomationService');
|
|
941
|
-
if (isAccessibilityConflict) {
|
|
942
|
-
debugDevice("YADB failed due to AccessibilityService conflict (likely Appium running), falling back to clipboard method");
|
|
943
|
-
await this.inputViaClipboard(keyboardContent);
|
|
944
|
-
} else debugDevice(`YADB execution may have completed despite error: ${error}`);
|
|
945
|
-
}
|
|
946
|
-
}
|
|
947
|
-
async inputViaClipboard(text) {
|
|
948
|
-
const adb = await this.getAdb();
|
|
949
|
-
try {
|
|
950
|
-
debugDevice(`Inputting via clipboard: "${text}"`);
|
|
951
|
-
const escapedText = text.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
952
|
-
const setClipboardCmd = `
|
|
953
|
-
content insert --uri content://settings/system --bind name:s:clipboard_text --bind value:s:"${escapedText}"
|
|
954
|
-
`;
|
|
955
|
-
await adb.shell(setClipboardCmd);
|
|
956
|
-
await sleep(100);
|
|
957
|
-
await adb.shell('input keyevent KEYCODE_PASTE');
|
|
958
|
-
await sleep(100);
|
|
959
|
-
debugDevice(`Clipboard input completed via content provider: "${text}"`);
|
|
960
|
-
} catch (error1) {
|
|
961
|
-
debugDevice(`Content provider clipboard failed, trying clipper app: ${error1}`);
|
|
962
|
-
try {
|
|
963
|
-
const base64Text = Buffer.from(text, 'utf-8').toString('base64');
|
|
964
|
-
await adb.shell(`am broadcast -a clipper.set -e text "${base64Text}"`);
|
|
965
|
-
await sleep(100);
|
|
966
|
-
await adb.shell('input keyevent KEYCODE_PASTE');
|
|
967
|
-
await sleep(100);
|
|
968
|
-
debugDevice(`Clipboard input completed via clipper: "${text}"`);
|
|
969
|
-
} catch (error2) {
|
|
970
|
-
debugDevice(`All clipboard methods failed: ${error2}`);
|
|
971
|
-
const isPureAscii = /^[\x00-\x7F]*$/.test(text);
|
|
972
|
-
if (isPureAscii) {
|
|
973
|
-
debugDevice(`Using ADB inputText for ASCII text: "${text}"`);
|
|
974
|
-
await adb.inputText(text);
|
|
975
|
-
} else await this.inputCharByChar(text);
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
}
|
|
979
|
-
async inputCharByChar(text) {
|
|
980
|
-
const adb = await this.getAdb();
|
|
981
|
-
debugDevice(`Inputting character by character (slow method): "${text}"`);
|
|
982
|
-
const chars = Array.from(text);
|
|
983
|
-
for (const char of chars){
|
|
984
|
-
if (' ' === char) await adb.shell('input keyevent KEYCODE_SPACE');
|
|
985
|
-
else await adb.shell(`input text "${char}"`);
|
|
986
|
-
await sleep(50);
|
|
987
|
-
}
|
|
988
|
-
debugDevice("Character-by-character input completed");
|
|
977
|
+
await adb.shell(`app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard '${keyboardContent}'`);
|
|
989
978
|
}
|
|
990
979
|
async getElementsInfo() {
|
|
991
980
|
return [];
|
|
@@ -998,6 +987,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
998
987
|
}
|
|
999
988
|
async getScreenSize() {
|
|
1000
989
|
const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
|
|
990
|
+
debugDevice(`getScreenSize: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedSize=${!!this.cachedScreenSize}`);
|
|
1001
991
|
if (shouldCache && this.cachedScreenSize) return this.cachedScreenSize;
|
|
1002
992
|
const adb = await this.getAdb();
|
|
1003
993
|
if ('number' == typeof this.options?.displayId) try {
|
|
@@ -1131,6 +1121,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1131
1121
|
}
|
|
1132
1122
|
async getDisplayOrientation() {
|
|
1133
1123
|
const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
|
|
1124
|
+
debugDevice(`getDisplayOrientation: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedOrientation=${null !== this.cachedOrientation}`);
|
|
1134
1125
|
if (shouldCache && null !== this.cachedOrientation) return this.cachedOrientation;
|
|
1135
1126
|
const adb = await this.getAdb();
|
|
1136
1127
|
let orientation = 0;
|
|
@@ -1156,6 +1147,15 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1156
1147
|
if (shouldCache) this.cachedOrientation = orientation;
|
|
1157
1148
|
return orientation;
|
|
1158
1149
|
}
|
|
1150
|
+
async getOrientedPhysicalSize() {
|
|
1151
|
+
const info = await this.getDevicePhysicalInfo();
|
|
1152
|
+
const isLandscape = 1 === info.orientation || 3 === info.orientation;
|
|
1153
|
+
const shouldSwap = true !== info.isCurrentOrientation && isLandscape;
|
|
1154
|
+
return {
|
|
1155
|
+
width: shouldSwap ? info.physicalHeight : info.physicalWidth,
|
|
1156
|
+
height: shouldSwap ? info.physicalWidth : info.physicalHeight
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
1159
1159
|
async size() {
|
|
1160
1160
|
const deviceInfo = await this.getDevicePhysicalInfo();
|
|
1161
1161
|
const adapter = this.getScrcpyAdapter();
|
|
@@ -1182,7 +1182,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1182
1182
|
height: logicalHeight
|
|
1183
1183
|
};
|
|
1184
1184
|
}
|
|
1185
|
-
async cacheFeatureForPoint(center
|
|
1185
|
+
async cacheFeatureForPoint(center) {
|
|
1186
1186
|
const { width, height } = await this.size();
|
|
1187
1187
|
debugDevice('cacheFeatureForPoint: center=[%s,%s], screen=[%s,%s]', center[0], center[1], width, height);
|
|
1188
1188
|
return {
|
|
@@ -1262,14 +1262,23 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1262
1262
|
const androidScreenshotPath = `/data/local/tmp/ms_${screenshotId}.png`;
|
|
1263
1263
|
const useShellScreencap = 'number' == typeof this.options?.displayId;
|
|
1264
1264
|
try {
|
|
1265
|
-
if (useShellScreencap
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1265
|
+
if (!useShellScreencap && this.takeScreenshotFailCount < AndroidDevice.TAKE_SCREENSHOT_FAIL_THRESHOLD) {
|
|
1266
|
+
debugDevice('Taking screenshot via adb.takeScreenshot');
|
|
1267
|
+
screenshotBuffer = await adb.takeScreenshot(null);
|
|
1268
|
+
debugDevice('adb.takeScreenshot completed');
|
|
1269
|
+
if (!screenshotBuffer) {
|
|
1270
|
+
this.takeScreenshotFailCount++;
|
|
1271
|
+
throw new Error('Failed to capture screenshot: screenshotBuffer is null');
|
|
1272
|
+
}
|
|
1273
|
+
if (!isValidImageBuffer(screenshotBuffer)) {
|
|
1274
|
+
debugDevice('Invalid image buffer detected: not a valid image format');
|
|
1275
|
+
this.takeScreenshotFailCount++;
|
|
1276
|
+
throw new Error('Screenshot buffer has invalid format: could not find valid image signature');
|
|
1277
|
+
}
|
|
1278
|
+
this.takeScreenshotFailCount = 0;
|
|
1279
|
+
} else {
|
|
1280
|
+
if (this.takeScreenshotFailCount >= AndroidDevice.TAKE_SCREENSHOT_FAIL_THRESHOLD) debugDevice('Skipping takeScreenshot (failed %d consecutive times), using shell screencap directly', this.takeScreenshotFailCount);
|
|
1281
|
+
throw new Error('Using shell screencap directly');
|
|
1273
1282
|
}
|
|
1274
1283
|
const validScreenshotBufferSize = this.options?.minScreenshotBufferSize ?? 10240;
|
|
1275
1284
|
if (validScreenshotBufferSize > 0 && screenshotBuffer.length < validScreenshotBufferSize) {
|
|
@@ -1298,12 +1307,21 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1298
1307
|
screenshotBuffer = await external_node_fs_["default"].promises.readFile(screenshotPath);
|
|
1299
1308
|
const validScreenshotBufferSize = this.options?.minScreenshotBufferSize ?? 10240;
|
|
1300
1309
|
if (!screenshotBuffer || validScreenshotBufferSize > 0 && screenshotBuffer.length < validScreenshotBufferSize) throw new Error(`Fallback screenshot validation failed: buffer size ${screenshotBuffer?.length || 0} bytes (minimum: ${validScreenshotBufferSize})`);
|
|
1301
|
-
if (!
|
|
1310
|
+
if (!isValidImageBuffer(screenshotBuffer)) throw new Error('Fallback screenshot buffer has invalid PNG format');
|
|
1302
1311
|
debugDevice(`Fallback screenshot validated successfully: ${screenshotBuffer.length} bytes`);
|
|
1303
1312
|
} finally{
|
|
1304
|
-
|
|
1305
|
-
|
|
1313
|
+
const adbPath = adb.executable?.path ?? 'adb';
|
|
1314
|
+
const child = execFile(adbPath, [
|
|
1315
|
+
'-s',
|
|
1316
|
+
this.deviceId,
|
|
1317
|
+
'shell',
|
|
1318
|
+
`rm ${androidScreenshotPath}`
|
|
1319
|
+
], {
|
|
1320
|
+
timeout: 3000
|
|
1321
|
+
}, (err)=>{
|
|
1322
|
+
if (err) debugDevice('Failed to delete remote screenshot: %s', err.message);
|
|
1306
1323
|
});
|
|
1324
|
+
child.unref();
|
|
1307
1325
|
}
|
|
1308
1326
|
}
|
|
1309
1327
|
if (!screenshotBuffer) throw new Error('Failed to capture screenshot: all methods failed');
|
|
@@ -1466,24 +1484,32 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1466
1484
|
if (!this.yadbPushed) {
|
|
1467
1485
|
const adb = await this.getAdb();
|
|
1468
1486
|
const androidPkgJson = (0, external_node_module_.createRequire)(import.meta.url).resolve('@aiscene/android/package.json');
|
|
1469
|
-
const
|
|
1470
|
-
await adb.push(
|
|
1487
|
+
const yadbBin = external_node_path_["default"].join(external_node_path_["default"].dirname(androidPkgJson), 'bin', 'yadb');
|
|
1488
|
+
await adb.push(yadbBin, '/data/local/tmp');
|
|
1471
1489
|
this.yadbPushed = true;
|
|
1472
1490
|
}
|
|
1473
1491
|
}
|
|
1474
1492
|
shouldUseYadbForText(text) {
|
|
1475
1493
|
const hasNonAscii = /[\x80-\uFFFF]/.test(text);
|
|
1476
1494
|
const hasFormatSpecifiers = /%[a-zA-Z]/.test(text);
|
|
1477
|
-
|
|
1495
|
+
const hasShellSpecialChars = /[\\`$]/.test(text);
|
|
1496
|
+
const hasBothQuotes = text.includes('"') && text.includes("'");
|
|
1497
|
+
return hasNonAscii || hasFormatSpecifiers || hasShellSpecialChars || hasBothQuotes;
|
|
1478
1498
|
}
|
|
1479
1499
|
async keyboardType(text, options) {
|
|
1480
1500
|
if (!text) return;
|
|
1481
1501
|
const adb = await this.getAdb();
|
|
1482
|
-
const shouldUseYadb = this.shouldUseYadbForText(text);
|
|
1483
1502
|
const IME_STRATEGY = (this.options?.imeStrategy || globalConfigManager.getEnvConfigValue(MIDSCENE_ANDROID_IME_STRATEGY)) ?? IME_STRATEGY_YADB_FOR_NON_ASCII;
|
|
1484
1503
|
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
1485
|
-
|
|
1486
|
-
|
|
1504
|
+
const useYadb = IME_STRATEGY === IME_STRATEGY_ALWAYS_YADB || IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII && this.shouldUseYadbForText(text);
|
|
1505
|
+
if (useYadb) await this.execYadb(escapeForShell(text));
|
|
1506
|
+
else {
|
|
1507
|
+
const segments = text.split('\n');
|
|
1508
|
+
for(let i = 0; i < segments.length; i++){
|
|
1509
|
+
if (segments[i].length > 0) await adb.inputText(segments[i]);
|
|
1510
|
+
if (i < segments.length - 1) await adb.keyevent(66);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1487
1513
|
if (true === shouldAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
1488
1514
|
}
|
|
1489
1515
|
normalizeKeyName(key) {
|
|
@@ -1531,12 +1557,12 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1531
1557
|
}
|
|
1532
1558
|
async mouseClick(x, y) {
|
|
1533
1559
|
const adb = await this.getAdb();
|
|
1534
|
-
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
1560
|
+
const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
|
|
1535
1561
|
await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} 150`);
|
|
1536
1562
|
}
|
|
1537
1563
|
async mouseDoubleClick(x, y) {
|
|
1538
1564
|
const adb = await this.getAdb();
|
|
1539
|
-
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
1565
|
+
const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
|
|
1540
1566
|
const tapCommand = `input${this.getDisplayArg()} tap ${adjustedX} ${adjustedY}`;
|
|
1541
1567
|
await adb.shell(tapCommand);
|
|
1542
1568
|
await sleep(50);
|
|
@@ -1547,8 +1573,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1547
1573
|
}
|
|
1548
1574
|
async mouseDrag(from, to, duration) {
|
|
1549
1575
|
const adb = await this.getAdb();
|
|
1550
|
-
const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
|
|
1551
|
-
const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
|
|
1576
|
+
const { x: fromX, y: fromY } = await this.adjustCoordinates(from.x, from.y);
|
|
1577
|
+
const { x: toX, y: toY } = await this.adjustCoordinates(to.x, to.y);
|
|
1552
1578
|
const swipeDuration = duration ?? defaultNormalScrollDuration;
|
|
1553
1579
|
await adb.shell(`input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`);
|
|
1554
1580
|
}
|
|
@@ -1558,16 +1584,16 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1558
1584
|
const n = 4;
|
|
1559
1585
|
const startX = Math.round(deltaX < 0 ? width / n * (n - 1) : width / n);
|
|
1560
1586
|
const startY = Math.round(deltaY < 0 ? height / n * (n - 1) : height / n);
|
|
1561
|
-
const
|
|
1562
|
-
const
|
|
1563
|
-
const
|
|
1564
|
-
const
|
|
1587
|
+
const maxPositiveDeltaX = startX;
|
|
1588
|
+
const maxNegativeDeltaX = width - startX;
|
|
1589
|
+
const maxPositiveDeltaY = startY;
|
|
1590
|
+
const maxNegativeDeltaY = height - startY;
|
|
1565
1591
|
deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
|
|
1566
1592
|
deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
|
|
1567
1593
|
const endX = Math.round(startX - deltaX);
|
|
1568
1594
|
const endY = Math.round(startY - deltaY);
|
|
1569
|
-
const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(startX, startY);
|
|
1570
|
-
const { x: adjustedEndX, y: adjustedEndY } = this.adjustCoordinates(endX, endY);
|
|
1595
|
+
const { x: adjustedStartX, y: adjustedStartY } = await this.adjustCoordinates(startX, startY);
|
|
1596
|
+
const { x: adjustedEndX, y: adjustedEndY } = await this.adjustCoordinates(endX, endY);
|
|
1571
1597
|
const adb = await this.getAdb();
|
|
1572
1598
|
const swipeDuration = duration ?? defaultNormalScrollDuration;
|
|
1573
1599
|
await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`);
|
|
@@ -1578,6 +1604,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1578
1604
|
this.cachedPhysicalDisplayId = void 0;
|
|
1579
1605
|
this.cachedScreenSize = null;
|
|
1580
1606
|
this.cachedOrientation = null;
|
|
1607
|
+
this.scalingRatio = 1;
|
|
1581
1608
|
if (this.scrcpyAdapter) {
|
|
1582
1609
|
await this.scrcpyAdapter.disconnect();
|
|
1583
1610
|
this.scrcpyAdapter = null;
|
|
@@ -1617,7 +1644,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1617
1644
|
}
|
|
1618
1645
|
async longPress(x, y, duration = 2000) {
|
|
1619
1646
|
const adb = await this.getAdb();
|
|
1620
|
-
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
1647
|
+
const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
|
|
1621
1648
|
await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} ${duration}`);
|
|
1622
1649
|
}
|
|
1623
1650
|
async pullDown(startPoint, distance, duration = 800) {
|
|
@@ -1639,8 +1666,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1639
1666
|
}
|
|
1640
1667
|
async pullDrag(from, to, duration) {
|
|
1641
1668
|
const adb = await this.getAdb();
|
|
1642
|
-
const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
|
|
1643
|
-
const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
|
|
1669
|
+
const { x: fromX, y: fromY } = await this.adjustCoordinates(from.x, from.y);
|
|
1670
|
+
const { x: toX, y: toY } = await this.adjustCoordinates(to.x, to.y);
|
|
1644
1671
|
await adb.shell(`input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${duration}`);
|
|
1645
1672
|
}
|
|
1646
1673
|
async pullUp(startPoint, distance, duration = 600) {
|
|
@@ -1727,7 +1754,6 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1727
1754
|
device_define_property(this, "yadbPushed", false);
|
|
1728
1755
|
device_define_property(this, "devicePixelRatio", 1);
|
|
1729
1756
|
device_define_property(this, "devicePixelRatioInitialized", false);
|
|
1730
|
-
device_define_property(this, "scalingRatio", 1);
|
|
1731
1757
|
device_define_property(this, "adb", null);
|
|
1732
1758
|
device_define_property(this, "connectingAdb", null);
|
|
1733
1759
|
device_define_property(this, "destroyed", false);
|
|
@@ -1738,6 +1764,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1738
1764
|
device_define_property(this, "cachedPhysicalDisplayId", void 0);
|
|
1739
1765
|
device_define_property(this, "scrcpyAdapter", null);
|
|
1740
1766
|
device_define_property(this, "appNameMapping", {});
|
|
1767
|
+
device_define_property(this, "scalingRatio", 1);
|
|
1768
|
+
device_define_property(this, "takeScreenshotFailCount", 0);
|
|
1741
1769
|
device_define_property(this, "interfaceType", 'android');
|
|
1742
1770
|
device_define_property(this, "uri", void 0);
|
|
1743
1771
|
device_define_property(this, "options", void 0);
|
|
@@ -1747,6 +1775,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1747
1775
|
this.customActions = options?.customActions;
|
|
1748
1776
|
}
|
|
1749
1777
|
}
|
|
1778
|
+
device_define_property(AndroidDevice, "TAKE_SCREENSHOT_FAIL_THRESHOLD", 3);
|
|
1750
1779
|
const runAdbShellParamSchema = z.object({
|
|
1751
1780
|
command: z.string().describe('ADB shell command to execute')
|
|
1752
1781
|
});
|
|
@@ -1759,6 +1788,9 @@ const createPlatformActions = (device)=>({
|
|
|
1759
1788
|
description: 'Execute ADB shell command on Android device',
|
|
1760
1789
|
interfaceAlias: 'runAdbShell',
|
|
1761
1790
|
paramSchema: runAdbShellParamSchema,
|
|
1791
|
+
sample: {
|
|
1792
|
+
command: 'dumpsys window displays | grep -E "mCurrentFocus"'
|
|
1793
|
+
},
|
|
1762
1794
|
call: async (param)=>{
|
|
1763
1795
|
if (!param.command || '' === param.command.trim()) throw new Error('RunAdbShell requires a non-empty command parameter');
|
|
1764
1796
|
const adb = await device.getAdb();
|
|
@@ -1770,6 +1802,9 @@ const createPlatformActions = (device)=>({
|
|
|
1770
1802
|
description: 'Launch an Android app or URL',
|
|
1771
1803
|
interfaceAlias: 'launch',
|
|
1772
1804
|
paramSchema: launchParamSchema,
|
|
1805
|
+
sample: {
|
|
1806
|
+
uri: 'com.example.app'
|
|
1807
|
+
},
|
|
1773
1808
|
call: async (param)=>{
|
|
1774
1809
|
if (!param.uri || '' === param.uri.trim()) throw new Error('Launch requires a non-empty uri parameter');
|
|
1775
1810
|
await device.launch(param.uri);
|
|
@@ -1922,7 +1957,7 @@ class AndroidMCPServer extends BaseMCPServer {
|
|
|
1922
1957
|
constructor(toolsManager){
|
|
1923
1958
|
super({
|
|
1924
1959
|
name: '@midscene/android-mcp',
|
|
1925
|
-
version: "1.6.
|
|
1960
|
+
version: "1.6.8",
|
|
1926
1961
|
description: 'Control the Android device using natural language commands'
|
|
1927
1962
|
}, toolsManager);
|
|
1928
1963
|
}
|