@midscene/android 1.9.5-beta-20260611045217.0 → 1.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +139 -132
- package/dist/es/index.mjs +136 -130
- package/dist/es/mcp-server.mjs +138 -131
- package/dist/lib/cli.js +67 -60
- package/dist/lib/index.js +66 -59
- package/dist/lib/mcp-server.js +67 -60
- package/dist/types/index.d.ts +0 -3
- package/package.json +4 -4
package/dist/es/index.mjs
CHANGED
|
@@ -2,29 +2,27 @@ import * as __rspack_external__midscene_shared_logger_b1dc2426 from "@midscene/s
|
|
|
2
2
|
import * as __rspack_external_node_fs_5ea92f0c from "node:fs";
|
|
3
3
|
import * as __rspack_external_node_module_ab9f2194 from "node:module";
|
|
4
4
|
import * as __rspack_external_node_path_c5b9b54f from "node:path";
|
|
5
|
-
import
|
|
6
|
-
import
|
|
7
|
-
import
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
import
|
|
11
|
-
import
|
|
12
|
-
import
|
|
13
|
-
import
|
|
14
|
-
import
|
|
15
|
-
import
|
|
16
|
-
import * as __rspack_external__midscene_core_agent_2b7f9158 from "@midscene/core/agent";
|
|
17
|
-
import * as __rspack_external__midscene_shared_mcp_base_tools_a0c805f3 from "@midscene/shared/mcp/base-tools";
|
|
5
|
+
import node_assert from "node:assert";
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
|
+
import { getMidsceneLocationSchema, z } from "@midscene/core";
|
|
8
|
+
import { createDefaultMobileActions, defineAction } from "@midscene/core/device";
|
|
9
|
+
import { getTmpFile, sleep } from "@midscene/core/utils";
|
|
10
|
+
import { MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, globalConfigManager, overrideAIConfig } from "@midscene/shared/env";
|
|
11
|
+
import { createImgBase64ByFormat, validateScreenshotBuffer } from "@midscene/shared/img";
|
|
12
|
+
import { mergeAndNormalizeAppNameMapping, normalizeForComparison, repeat } from "@midscene/shared/utils";
|
|
13
|
+
import { ADB } from "appium-adb";
|
|
14
|
+
import { Agent } from "@midscene/core/agent";
|
|
15
|
+
import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
|
|
18
16
|
var __webpack_modules__ = {
|
|
19
17
|
"./src/scrcpy-manager.ts" (__unused_rspack_module, __webpack_exports__, __webpack_require__) {
|
|
20
18
|
__webpack_require__.d(__webpack_exports__, {
|
|
21
19
|
ScrcpyScreenshotManager: ()=>ScrcpyScreenshotManager,
|
|
22
20
|
o: ()=>DEFAULT_SCRCPY_CONFIG
|
|
23
21
|
});
|
|
24
|
-
var
|
|
25
|
-
var
|
|
26
|
-
var
|
|
27
|
-
var
|
|
22
|
+
var node_fs__rspack_import_0 = __webpack_require__("node:fs");
|
|
23
|
+
var node_module__rspack_import_1 = __webpack_require__("node:module");
|
|
24
|
+
var node_path__rspack_import_2 = __webpack_require__("node:path");
|
|
25
|
+
var _midscene_shared_logger__rspack_import_3 = __webpack_require__("@midscene/shared/logger");
|
|
28
26
|
function _define_property(obj, key, value) {
|
|
29
27
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
30
28
|
value: value,
|
|
@@ -35,8 +33,8 @@ var __webpack_modules__ = {
|
|
|
35
33
|
else obj[key] = value;
|
|
36
34
|
return obj;
|
|
37
35
|
}
|
|
38
|
-
const debugScrcpy = (0,
|
|
39
|
-
const warnScrcpy = (0,
|
|
36
|
+
const debugScrcpy = (0, _midscene_shared_logger__rspack_import_3.getDebug)('android:scrcpy');
|
|
37
|
+
const warnScrcpy = (0, _midscene_shared_logger__rspack_import_3.getDebug)('android:scrcpy', {
|
|
40
38
|
console: true
|
|
41
39
|
});
|
|
42
40
|
const NAL_TYPE_IDR = 5;
|
|
@@ -77,11 +75,6 @@ var __webpack_modules__ = {
|
|
|
77
75
|
return false;
|
|
78
76
|
}
|
|
79
77
|
class ScrcpyScreenshotManager {
|
|
80
|
-
shouldRetryWithForwardTunnel(error) {
|
|
81
|
-
const cause = error instanceof Error ? error.cause : void 0;
|
|
82
|
-
const message = error instanceof Error ? `${error.message}${cause instanceof Error ? ` ${cause.message}` : ''}` : String(error);
|
|
83
|
-
return /more than one device\/emulator/i.test(message);
|
|
84
|
-
}
|
|
85
78
|
async validateEnvironment() {
|
|
86
79
|
await this.ensureFfmpegAvailable();
|
|
87
80
|
}
|
|
@@ -100,7 +93,12 @@ var __webpack_modules__ = {
|
|
|
100
93
|
try {
|
|
101
94
|
this.isConnecting = true;
|
|
102
95
|
debugScrcpy('Starting scrcpy connection...');
|
|
103
|
-
|
|
96
|
+
const { AdbScrcpyClient, AdbScrcpyOptions3_3_3 } = await import("@yume-chan/adb-scrcpy");
|
|
97
|
+
const { ReadableStream } = await import("@yume-chan/stream-extra");
|
|
98
|
+
const { DefaultServerPath } = await import("@yume-chan/scrcpy");
|
|
99
|
+
const serverBinPath = this.resolveServerBinPath();
|
|
100
|
+
await AdbScrcpyClient.pushServer(this.adb, ReadableStream.from((0, node_fs__rspack_import_0.createReadStream)(serverBinPath)));
|
|
101
|
+
const scrcpyOptions = new AdbScrcpyOptions3_3_3({
|
|
104
102
|
audio: false,
|
|
105
103
|
control: false,
|
|
106
104
|
maxSize: this.options.maxSize,
|
|
@@ -109,6 +107,7 @@ var __webpack_modules__ = {
|
|
|
109
107
|
sendFrameMeta: true,
|
|
110
108
|
videoCodecOptions: 'i-frame-interval=0,bitrate-mode=2'
|
|
111
109
|
});
|
|
110
|
+
this.scrcpyClient = await AdbScrcpyClient.start(this.adb, DefaultServerPath, scrcpyOptions);
|
|
112
111
|
const videoStreamPromise = this.scrcpyClient.videoStream;
|
|
113
112
|
if (!videoStreamPromise) throw new Error('Scrcpy client did not provide video stream');
|
|
114
113
|
this.videoStream = await videoStreamPromise;
|
|
@@ -130,50 +129,13 @@ var __webpack_modules__ = {
|
|
|
130
129
|
this.isConnecting = false;
|
|
131
130
|
}
|
|
132
131
|
}
|
|
133
|
-
async startScrcpyOnce(adb, options = {}, onProgress, tunnelForward = false) {
|
|
134
|
-
const { AdbScrcpyClient, AdbScrcpyOptions3_3_3 } = await import("@yume-chan/adb-scrcpy");
|
|
135
|
-
const { ReadableStream } = await import("@yume-chan/stream-extra");
|
|
136
|
-
const { DefaultServerPath } = await import("@yume-chan/scrcpy");
|
|
137
|
-
const serverBinPath = this.resolveServerBinPath();
|
|
138
|
-
onProgress?.('pushing-server');
|
|
139
|
-
await (0, __rspack_external__midscene_shared_timeout_982edd16.withTimeout)(AdbScrcpyClient.pushServer(adb, ReadableStream.from((0, external_node_fs_.createReadStream)(serverBinPath))), __rspack_external__midscene_shared_constants_ab069bca.SCRCPY_PUSH_TIMEOUT_MS, `Timed out pushing scrcpy server to device after ${Math.round(__rspack_external__midscene_shared_constants_ab069bca.SCRCPY_PUSH_TIMEOUT_MS / 1000)}s`);
|
|
140
|
-
const scrcpyOptions = new AdbScrcpyOptions3_3_3({
|
|
141
|
-
audio: false,
|
|
142
|
-
control: true,
|
|
143
|
-
maxSize: 1024,
|
|
144
|
-
sendFrameMeta: true,
|
|
145
|
-
videoBitRate: 2000000,
|
|
146
|
-
...options,
|
|
147
|
-
tunnelForward
|
|
148
|
-
});
|
|
149
|
-
onProgress?.('starting-service');
|
|
150
|
-
const startPromise = AdbScrcpyClient.start(adb, DefaultServerPath, scrcpyOptions);
|
|
151
|
-
return await (0, __rspack_external__midscene_shared_timeout_982edd16.withTimeout)(startPromise, __rspack_external__midscene_shared_constants_ab069bca.SCRCPY_START_TIMEOUT_MS, `Timed out starting scrcpy service after ${Math.round(__rspack_external__midscene_shared_constants_ab069bca.SCRCPY_START_TIMEOUT_MS / 1000)}s`, {
|
|
152
|
-
onSettledAfterTimeout: async (lateClient)=>{
|
|
153
|
-
try {
|
|
154
|
-
await lateClient.close();
|
|
155
|
-
} catch (closeError) {
|
|
156
|
-
console.error('failed to close late scrcpy client after timeout:', closeError);
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
async startScrcpy(adb, options = {}, onProgress) {
|
|
162
|
-
try {
|
|
163
|
-
return await this.startScrcpyOnce(adb, options, onProgress, false);
|
|
164
|
-
} catch (error) {
|
|
165
|
-
if (!this.shouldRetryWithForwardTunnel(error)) throw error;
|
|
166
|
-
warnScrcpy(`Reverse tunnel failed for device ${adb.serial}; retrying scrcpy with forward tunnel: ${error}`);
|
|
167
|
-
return await this.startScrcpyOnce(adb, options, onProgress, true);
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
132
|
resolveServerBinPath() {
|
|
171
|
-
const androidPkgJson = (0,
|
|
172
|
-
return
|
|
133
|
+
const androidPkgJson = (0, node_module__rspack_import_1.createRequire)(import.meta.url).resolve('@midscene/android/package.json');
|
|
134
|
+
return node_path__rspack_import_2["default"].join(node_path__rspack_import_2["default"].dirname(androidPkgJson), 'bin', 'scrcpy-server');
|
|
173
135
|
}
|
|
174
136
|
getFfmpegPath() {
|
|
175
137
|
try {
|
|
176
|
-
const dynamicRequire = (0,
|
|
138
|
+
const dynamicRequire = (0, node_module__rspack_import_1.createRequire)(import.meta.url);
|
|
177
139
|
const ffmpegInstaller = dynamicRequire('@ffmpeg-installer/ffmpeg');
|
|
178
140
|
debugScrcpy(`Using ffmpeg from npm package: ${ffmpegInstaller.path}`);
|
|
179
141
|
return ffmpegInstaller.path;
|
|
@@ -487,6 +449,45 @@ var external_node_fs_ = __webpack_require__("node:fs");
|
|
|
487
449
|
var external_node_module_ = __webpack_require__("node:module");
|
|
488
450
|
var external_node_path_ = __webpack_require__("node:path");
|
|
489
451
|
var logger_ = __webpack_require__("@midscene/shared/logger");
|
|
452
|
+
const EMPTY_ADB_SHELL_STDOUT = '<empty>';
|
|
453
|
+
const MAX_RUN_ADB_SHELL_STDOUT = 200;
|
|
454
|
+
function normalizeShellStream(value) {
|
|
455
|
+
if (null == value) return '';
|
|
456
|
+
return String(value);
|
|
457
|
+
}
|
|
458
|
+
function truncateAdbShellStream(output, streamName) {
|
|
459
|
+
if (output.length <= MAX_RUN_ADB_SHELL_STDOUT) return output;
|
|
460
|
+
return `${output.slice(0, MAX_RUN_ADB_SHELL_STDOUT)}
|
|
461
|
+
...[${streamName} truncated, ${output.length - MAX_RUN_ADB_SHELL_STDOUT} more characters]`;
|
|
462
|
+
}
|
|
463
|
+
function buildRunAdbShellPlanningFeedback({ command, stdout }) {
|
|
464
|
+
if ('' === stdout) return;
|
|
465
|
+
const commandText = 'string' == typeof command && command.length > 0 ? `Command: ${command}\n` : '';
|
|
466
|
+
return `${commandText}Stdout:
|
|
467
|
+
${stdout}`;
|
|
468
|
+
}
|
|
469
|
+
function buildAdbShellStderrErrorMessage(command, stdout, stderr) {
|
|
470
|
+
return `RunAdbShell command returned stderr.
|
|
471
|
+
Command: ${command}
|
|
472
|
+
Stderr:
|
|
473
|
+
${truncateAdbShellStream(stderr, 'stderr')}
|
|
474
|
+
Stdout:
|
|
475
|
+
${stdout ? truncateAdbShellStream(stdout, 'stdout') : EMPTY_ADB_SHELL_STDOUT}`;
|
|
476
|
+
}
|
|
477
|
+
function getAdbShellStdoutOrThrow(command, output) {
|
|
478
|
+
if ('string' == typeof output) return output;
|
|
479
|
+
const stdout = normalizeShellStream(output.stdout);
|
|
480
|
+
const stderr = normalizeShellStream(output.stderr);
|
|
481
|
+
if (stderr) throw new Error(buildAdbShellStderrErrorMessage(command, stdout, stderr));
|
|
482
|
+
return stdout;
|
|
483
|
+
}
|
|
484
|
+
async function runAdbShellStdoutOrThrow(adb, command, options = {}) {
|
|
485
|
+
const output = await adb.shell(command, {
|
|
486
|
+
...options,
|
|
487
|
+
outputFormat: adb.EXEC_OUTPUT_FORMAT.FULL
|
|
488
|
+
});
|
|
489
|
+
return getAdbShellStdoutOrThrow(command, output);
|
|
490
|
+
}
|
|
490
491
|
var scrcpy_manager = __webpack_require__("./src/scrcpy-manager.ts");
|
|
491
492
|
function _define_property(obj, key, value) {
|
|
492
493
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
@@ -558,7 +559,7 @@ class ScrcpyDeviceAdapter {
|
|
|
558
559
|
async screenshotBase64(deviceInfo) {
|
|
559
560
|
const manager = await this.ensureManager(deviceInfo);
|
|
560
561
|
const screenshotBuffer = await manager.getScreenshotJpeg();
|
|
561
|
-
return
|
|
562
|
+
return createImgBase64ByFormat('jpeg', screenshotBuffer.toString('base64'));
|
|
562
563
|
}
|
|
563
564
|
getResolution() {
|
|
564
565
|
return this.manager?.getResolution() ?? null;
|
|
@@ -629,7 +630,7 @@ class AndroidDevice {
|
|
|
629
630
|
input: this.inputPrimitives,
|
|
630
631
|
size: ()=>this.size(),
|
|
631
632
|
sleep: async (timeMs)=>{
|
|
632
|
-
await
|
|
633
|
+
await sleep(timeMs);
|
|
633
634
|
},
|
|
634
635
|
getDefaultAutoDismissKeyboard: ()=>this.options?.autoDismissKeyboard,
|
|
635
636
|
systemActions: {
|
|
@@ -648,18 +649,18 @@ class AndroidDevice {
|
|
|
648
649
|
}
|
|
649
650
|
};
|
|
650
651
|
const defaultActions = [
|
|
651
|
-
...
|
|
652
|
-
|
|
652
|
+
...createDefaultMobileActions(mobileActionContext),
|
|
653
|
+
defineAction({
|
|
653
654
|
name: 'PullGesture',
|
|
654
655
|
description: 'Trigger pull down to refresh or pull up actions',
|
|
655
|
-
paramSchema:
|
|
656
|
-
direction:
|
|
656
|
+
paramSchema: z.object({
|
|
657
|
+
direction: z["enum"]([
|
|
657
658
|
'up',
|
|
658
659
|
'down'
|
|
659
660
|
]).describe('The direction to pull'),
|
|
660
|
-
distance:
|
|
661
|
-
duration:
|
|
662
|
-
locate:
|
|
661
|
+
distance: z.number().optional().describe('The distance to pull (in pixels)'),
|
|
662
|
+
duration: z.number().optional().describe('The duration of the pull (in milliseconds)'),
|
|
663
|
+
locate: getMidsceneLocationSchema().optional().describe('The element to start the pull from (optional)')
|
|
663
664
|
}),
|
|
664
665
|
sample: {
|
|
665
666
|
direction: 'down',
|
|
@@ -706,7 +707,7 @@ class AndroidDevice {
|
|
|
706
707
|
else if ('right' === param.direction) await this.scrollRight(param.distance || void 0, startingPoint);
|
|
707
708
|
else throw new Error(`Unknown scroll direction: ${param.direction}`);
|
|
708
709
|
else await this.scrollDown(param?.distance || void 0, startingPoint);
|
|
709
|
-
await
|
|
710
|
+
await sleep(500);
|
|
710
711
|
}
|
|
711
712
|
}
|
|
712
713
|
describe() {
|
|
@@ -733,10 +734,10 @@ class AndroidDevice {
|
|
|
733
734
|
let error = null;
|
|
734
735
|
debugDevice(`Initializing ADB with device ID: ${this.deviceId}`);
|
|
735
736
|
try {
|
|
736
|
-
const androidAdbPath = this.options?.androidAdbPath ||
|
|
737
|
-
const remoteAdbHost = this.options?.remoteAdbHost ||
|
|
738
|
-
const remoteAdbPort = this.options?.remoteAdbPort ||
|
|
739
|
-
this.adb = new
|
|
737
|
+
const androidAdbPath = this.options?.androidAdbPath || globalConfigManager.getEnvConfigValue(MIDSCENE_ADB_PATH);
|
|
738
|
+
const remoteAdbHost = this.options?.remoteAdbHost || globalConfigManager.getEnvConfigValue(MIDSCENE_ADB_REMOTE_HOST);
|
|
739
|
+
const remoteAdbPort = this.options?.remoteAdbPort || globalConfigManager.getEnvConfigValue(MIDSCENE_ADB_REMOTE_PORT);
|
|
740
|
+
this.adb = new ADB({
|
|
740
741
|
udid: this.deviceId,
|
|
741
742
|
adbExecTimeout: 60000,
|
|
742
743
|
executable: androidAdbPath ? {
|
|
@@ -810,7 +811,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
810
811
|
this.appNameMapping = mapping;
|
|
811
812
|
}
|
|
812
813
|
resolvePackageName(appName) {
|
|
813
|
-
const normalizedAppName =
|
|
814
|
+
const normalizedAppName = normalizeForComparison(appName);
|
|
814
815
|
return this.appNameMapping[normalizedAppName];
|
|
815
816
|
}
|
|
816
817
|
async launch(uri) {
|
|
@@ -1129,7 +1130,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1129
1130
|
screenshotBuffer = await adb.takeScreenshot.call(adb);
|
|
1130
1131
|
debugDevice('adb.takeScreenshot completed');
|
|
1131
1132
|
try {
|
|
1132
|
-
|
|
1133
|
+
validateScreenshotBuffer(screenshotBuffer, {
|
|
1133
1134
|
label: 'Screenshot',
|
|
1134
1135
|
minBufferSize: this.options?.minScreenshotBufferSize ?? AndroidDevice.DEFAULT_MIN_SCREENSHOT_BUFFER_SIZE
|
|
1135
1136
|
});
|
|
@@ -1145,7 +1146,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1145
1146
|
}
|
|
1146
1147
|
} catch (error) {
|
|
1147
1148
|
debugDevice(`Taking screenshot via adb.takeScreenshot failed or was skipped: ${error}`);
|
|
1148
|
-
const screenshotPath =
|
|
1149
|
+
const screenshotPath = getTmpFile('png');
|
|
1149
1150
|
localScreenshotPath = screenshotPath;
|
|
1150
1151
|
try {
|
|
1151
1152
|
debugDevice('Fallback: taking screenshot via shell screencap');
|
|
@@ -1163,14 +1164,14 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1163
1164
|
await adb.pull(androidScreenshotPath, screenshotPath);
|
|
1164
1165
|
debugDevice(`adb.pull completed, local path: ${screenshotPath}`);
|
|
1165
1166
|
screenshotBuffer = await external_node_fs_["default"].promises.readFile(screenshotPath);
|
|
1166
|
-
|
|
1167
|
+
validateScreenshotBuffer(screenshotBuffer, {
|
|
1167
1168
|
label: 'Fallback screenshot',
|
|
1168
1169
|
minBufferSize: this.options?.minScreenshotBufferSize ?? AndroidDevice.DEFAULT_MIN_SCREENSHOT_BUFFER_SIZE
|
|
1169
1170
|
});
|
|
1170
1171
|
debugDevice(`Fallback screenshot validated successfully: ${screenshotBuffer.length} bytes`);
|
|
1171
1172
|
} finally{
|
|
1172
1173
|
const adbPath = adb.executable?.path ?? 'adb';
|
|
1173
|
-
const child =
|
|
1174
|
+
const child = execFile(adbPath, [
|
|
1174
1175
|
'-s',
|
|
1175
1176
|
this.deviceId,
|
|
1176
1177
|
'shell',
|
|
@@ -1185,7 +1186,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1185
1186
|
}
|
|
1186
1187
|
if (!screenshotBuffer) throw new Error('Failed to capture screenshot: all methods failed');
|
|
1187
1188
|
debugDevice('Converting to base64');
|
|
1188
|
-
const result =
|
|
1189
|
+
const result = createImgBase64ByFormat('png', screenshotBuffer.toString('base64'));
|
|
1189
1190
|
if (localScreenshotPath) {
|
|
1190
1191
|
debugDevice(`Deleting local screenshot: ${localScreenshotPath}`);
|
|
1191
1192
|
(0, external_node_fs_.unlink)(localScreenshotPath, (unlinkError)=>{
|
|
@@ -1202,7 +1203,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1202
1203
|
});
|
|
1203
1204
|
await this.ensureYadb();
|
|
1204
1205
|
const adb = await this.getAdb();
|
|
1205
|
-
const IME_STRATEGY = (this.options?.imeStrategy ||
|
|
1206
|
+
const IME_STRATEGY = (this.options?.imeStrategy || globalConfigManager.getEnvConfigValue(MIDSCENE_ANDROID_IME_STRATEGY)) ?? IME_STRATEGY_YADB_FOR_NON_ASCII;
|
|
1206
1207
|
if (IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII) await adb.clearTextField(100);
|
|
1207
1208
|
else await adb.shell(`app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboardClear`);
|
|
1208
1209
|
if (await adb.isSoftKeyboardPresent()) return;
|
|
@@ -1230,12 +1231,12 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1230
1231
|
x: start.x,
|
|
1231
1232
|
y: Math.round(height)
|
|
1232
1233
|
};
|
|
1233
|
-
await
|
|
1234
|
-
await
|
|
1234
|
+
await repeat(defaultScrollUntilTimes, ()=>this.dragPoint(start, end, defaultFastScrollDuration));
|
|
1235
|
+
await sleep(1000);
|
|
1235
1236
|
return;
|
|
1236
1237
|
}
|
|
1237
|
-
await
|
|
1238
|
-
await
|
|
1238
|
+
await repeat(defaultScrollUntilTimes, ()=>this.scroll(0, -9999999, defaultFastScrollDuration));
|
|
1239
|
+
await sleep(1000);
|
|
1239
1240
|
}
|
|
1240
1241
|
async scrollUntilBottom(startPoint) {
|
|
1241
1242
|
if (startPoint) {
|
|
@@ -1247,12 +1248,12 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1247
1248
|
x: start.x,
|
|
1248
1249
|
y: 0
|
|
1249
1250
|
};
|
|
1250
|
-
await
|
|
1251
|
-
await
|
|
1251
|
+
await repeat(defaultScrollUntilTimes, ()=>this.dragPoint(start, end, defaultFastScrollDuration));
|
|
1252
|
+
await sleep(1000);
|
|
1252
1253
|
return;
|
|
1253
1254
|
}
|
|
1254
|
-
await
|
|
1255
|
-
await
|
|
1255
|
+
await repeat(defaultScrollUntilTimes, ()=>this.scroll(0, 9999999, defaultFastScrollDuration));
|
|
1256
|
+
await sleep(1000);
|
|
1256
1257
|
}
|
|
1257
1258
|
async scrollUntilLeft(startPoint) {
|
|
1258
1259
|
if (startPoint) {
|
|
@@ -1265,12 +1266,12 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1265
1266
|
x: Math.round(width),
|
|
1266
1267
|
y: start.y
|
|
1267
1268
|
};
|
|
1268
|
-
await
|
|
1269
|
-
await
|
|
1269
|
+
await repeat(defaultScrollUntilTimes, ()=>this.dragPoint(start, end, defaultFastScrollDuration));
|
|
1270
|
+
await sleep(1000);
|
|
1270
1271
|
return;
|
|
1271
1272
|
}
|
|
1272
|
-
await
|
|
1273
|
-
await
|
|
1273
|
+
await repeat(defaultScrollUntilTimes, ()=>this.scroll(-9999999, 0, defaultFastScrollDuration));
|
|
1274
|
+
await sleep(1000);
|
|
1274
1275
|
}
|
|
1275
1276
|
async scrollUntilRight(startPoint) {
|
|
1276
1277
|
if (startPoint) {
|
|
@@ -1282,12 +1283,12 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1282
1283
|
x: 0,
|
|
1283
1284
|
y: start.y
|
|
1284
1285
|
};
|
|
1285
|
-
await
|
|
1286
|
-
await
|
|
1286
|
+
await repeat(defaultScrollUntilTimes, ()=>this.dragPoint(start, end, defaultFastScrollDuration));
|
|
1287
|
+
await sleep(1000);
|
|
1287
1288
|
return;
|
|
1288
1289
|
}
|
|
1289
|
-
await
|
|
1290
|
-
await
|
|
1290
|
+
await repeat(defaultScrollUntilTimes, ()=>this.scroll(9999999, 0, defaultFastScrollDuration));
|
|
1291
|
+
await sleep(1000);
|
|
1291
1292
|
}
|
|
1292
1293
|
async scrollUp(distance, startPoint) {
|
|
1293
1294
|
const { height } = await this.size();
|
|
@@ -1372,7 +1373,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1372
1373
|
async typeText(text, options) {
|
|
1373
1374
|
if (!text) return;
|
|
1374
1375
|
const adb = await this.getAdb();
|
|
1375
|
-
const IME_STRATEGY = (this.options?.imeStrategy ||
|
|
1376
|
+
const IME_STRATEGY = (this.options?.imeStrategy || globalConfigManager.getEnvConfigValue(MIDSCENE_ANDROID_IME_STRATEGY)) ?? IME_STRATEGY_YADB_FOR_NON_ASCII;
|
|
1376
1377
|
const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
|
|
1377
1378
|
const useYadb = IME_STRATEGY === IME_STRATEGY_ALWAYS_YADB || IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII && this.shouldUseYadbForText(text);
|
|
1378
1379
|
if (useYadb) await this.execYadb(escapeForShell(text));
|
|
@@ -1438,7 +1439,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1438
1439
|
const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(point.x, point.y);
|
|
1439
1440
|
const tapCommand = `input${this.getDisplayArg()} tap ${adjustedX} ${adjustedY}`;
|
|
1440
1441
|
await adb.shell(tapCommand);
|
|
1441
|
-
await
|
|
1442
|
+
await sleep(50);
|
|
1442
1443
|
await adb.shell(tapCommand);
|
|
1443
1444
|
}
|
|
1444
1445
|
async mouseMove() {
|
|
@@ -1549,7 +1550,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1549
1550
|
y: start.y + pullDistance
|
|
1550
1551
|
};
|
|
1551
1552
|
await this.pullDrag(start, end, duration);
|
|
1552
|
-
await
|
|
1553
|
+
await sleep(200);
|
|
1553
1554
|
}
|
|
1554
1555
|
async pullDrag(from, to, duration) {
|
|
1555
1556
|
await this.swipePoint(from, to, duration);
|
|
@@ -1569,7 +1570,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1569
1570
|
y: start.y - pullDistance
|
|
1570
1571
|
};
|
|
1571
1572
|
await this.pullDrag(start, end, duration);
|
|
1572
|
-
await
|
|
1573
|
+
await sleep(100);
|
|
1573
1574
|
}
|
|
1574
1575
|
getDisplayArg() {
|
|
1575
1576
|
return 'number' == typeof this.options?.displayId ? ` -d ${this.options.displayId}` : '';
|
|
@@ -1620,7 +1621,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1620
1621
|
const startTime = Date.now();
|
|
1621
1622
|
const intervalMs = 100;
|
|
1622
1623
|
while(Date.now() - startTime < timeoutMs){
|
|
1623
|
-
await
|
|
1624
|
+
await sleep(intervalMs);
|
|
1624
1625
|
const currentStatus = await adb.isSoftKeyboardPresent();
|
|
1625
1626
|
const isStillShown = 'boolean' == typeof currentStatus ? currentStatus : currentStatus?.isKeyboardShown;
|
|
1626
1627
|
if (!isStillShown) {
|
|
@@ -1703,7 +1704,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1703
1704
|
recentAppsButton: ()=>this.recentApps()
|
|
1704
1705
|
}
|
|
1705
1706
|
});
|
|
1706
|
-
(
|
|
1707
|
+
node_assert(deviceId, 'deviceId is required for AndroidDevice');
|
|
1707
1708
|
this.deviceId = deviceId;
|
|
1708
1709
|
this.options = options;
|
|
1709
1710
|
this.customActions = options?.customActions;
|
|
@@ -1711,31 +1712,37 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1711
1712
|
}
|
|
1712
1713
|
device_define_property(AndroidDevice, "TAKE_SCREENSHOT_FAIL_THRESHOLD", 3);
|
|
1713
1714
|
device_define_property(AndroidDevice, "DEFAULT_MIN_SCREENSHOT_BUFFER_SIZE", 1024);
|
|
1714
|
-
const runAdbShellParamSchema =
|
|
1715
|
-
command:
|
|
1715
|
+
const runAdbShellParamSchema = z.object({
|
|
1716
|
+
command: z.string().describe('ADB shell command to execute')
|
|
1716
1717
|
});
|
|
1717
|
-
const launchParamSchema =
|
|
1718
|
-
uri:
|
|
1718
|
+
const launchParamSchema = z.object({
|
|
1719
|
+
uri: z.string().describe('App name, package name, or URL to launch. Prioritize using the exact package name or URL the user has provided. If none provided, use the accurate app name.')
|
|
1719
1720
|
});
|
|
1720
|
-
const terminateParamSchema =
|
|
1721
|
-
uri:
|
|
1721
|
+
const terminateParamSchema = z.object({
|
|
1722
|
+
uri: z.string().describe('Package name or app name to terminate. Use the exact package name, e.g. com.android.settings.')
|
|
1722
1723
|
});
|
|
1723
1724
|
const createPlatformActions = (device)=>({
|
|
1724
|
-
RunAdbShell:
|
|
1725
|
+
RunAdbShell: defineAction({
|
|
1725
1726
|
name: 'RunAdbShell',
|
|
1726
|
-
description: 'Execute ADB shell command on Android device',
|
|
1727
|
+
description: 'Execute an ADB shell command on the Android device and return the command stdout. Read the returned stdout to decide the next step; the stdout may indicate either success or failure.',
|
|
1727
1728
|
interfaceAlias: 'runAdbShell',
|
|
1728
1729
|
paramSchema: runAdbShellParamSchema,
|
|
1729
1730
|
sample: {
|
|
1730
1731
|
command: 'dumpsys window displays | grep -E "mCurrentFocus"'
|
|
1731
1732
|
},
|
|
1732
|
-
call: async (param)=>{
|
|
1733
|
+
call: async (param, context)=>{
|
|
1733
1734
|
if (!param.command || '' === param.command.trim()) throw new Error('RunAdbShell requires a non-empty command parameter');
|
|
1734
1735
|
const adb = await device.getAdb();
|
|
1735
|
-
|
|
1736
|
+
const stdout = await runAdbShellStdoutOrThrow(adb, param.command);
|
|
1737
|
+
const planningFeedback = buildRunAdbShellPlanningFeedback({
|
|
1738
|
+
command: param.command,
|
|
1739
|
+
stdout
|
|
1740
|
+
});
|
|
1741
|
+
if (planningFeedback && context?.task) context.task.planningFeedback = planningFeedback;
|
|
1742
|
+
return stdout;
|
|
1736
1743
|
}
|
|
1737
1744
|
}),
|
|
1738
|
-
Launch:
|
|
1745
|
+
Launch: defineAction({
|
|
1739
1746
|
name: 'Launch',
|
|
1740
1747
|
description: 'Launch an Android app or URL',
|
|
1741
1748
|
interfaceAlias: 'launch',
|
|
@@ -1748,7 +1755,7 @@ const createPlatformActions = (device)=>({
|
|
|
1748
1755
|
await device.launch(param.uri);
|
|
1749
1756
|
}
|
|
1750
1757
|
}),
|
|
1751
|
-
Terminate:
|
|
1758
|
+
Terminate: defineAction({
|
|
1752
1759
|
name: 'Terminate',
|
|
1753
1760
|
description: 'Terminate (force-stop) an Android app by package name',
|
|
1754
1761
|
interfaceAlias: 'terminate',
|
|
@@ -1884,13 +1891,13 @@ async function withTimeout(promise, timeoutMs, label) {
|
|
|
1884
1891
|
}
|
|
1885
1892
|
}
|
|
1886
1893
|
async function createAdbForDetailLookup() {
|
|
1887
|
-
return await
|
|
1894
|
+
return await ADB.createADB({
|
|
1888
1895
|
adbExecTimeout: DETAIL_LOOKUP_ADB_TIMEOUT_MS
|
|
1889
1896
|
});
|
|
1890
1897
|
}
|
|
1891
1898
|
async function getConnectedDevices() {
|
|
1892
1899
|
try {
|
|
1893
|
-
const adb = await
|
|
1900
|
+
const adb = await ADB.createADB({
|
|
1894
1901
|
adbExecTimeout: 60000
|
|
1895
1902
|
});
|
|
1896
1903
|
const devices = await adb.getConnectedDevices();
|
|
@@ -1949,7 +1956,7 @@ function agent_define_property(obj, key, value) {
|
|
|
1949
1956
|
return obj;
|
|
1950
1957
|
}
|
|
1951
1958
|
const debugAgent = (0, logger_.getDebug)('android:agent');
|
|
1952
|
-
class AndroidAgent extends
|
|
1959
|
+
class AndroidAgent extends Agent {
|
|
1953
1960
|
async launch(uri) {
|
|
1954
1961
|
const action = this.wrapActionInActionSpace('Launch');
|
|
1955
1962
|
return action({
|
|
@@ -1965,7 +1972,7 @@ class AndroidAgent extends __rspack_external__midscene_core_agent_2b7f9158.Agent
|
|
|
1965
1972
|
async runAdbShell(command, opt) {
|
|
1966
1973
|
if (opt?.timeout !== void 0) {
|
|
1967
1974
|
const adb = await this.interface.getAdb();
|
|
1968
|
-
return await adb
|
|
1975
|
+
return await runAdbShellStdoutOrThrow(adb, command, {
|
|
1969
1976
|
timeout: opt.timeout
|
|
1970
1977
|
});
|
|
1971
1978
|
}
|
|
@@ -1980,7 +1987,7 @@ class AndroidAgent extends __rspack_external__midscene_core_agent_2b7f9158.Agent
|
|
|
1980
1987
|
}
|
|
1981
1988
|
constructor(device, opts){
|
|
1982
1989
|
super(device, opts), agent_define_property(this, "back", void 0), agent_define_property(this, "home", void 0), agent_define_property(this, "recentApps", void 0), agent_define_property(this, "appNameMapping", void 0);
|
|
1983
|
-
this.appNameMapping =
|
|
1990
|
+
this.appNameMapping = mergeAndNormalizeAppNameMapping(defaultAppNameMapping, opts?.appNameMapping);
|
|
1984
1991
|
device.setAppNameMapping(this.appNameMapping);
|
|
1985
1992
|
this.back = this.createActionWrapper('AndroidBackButton');
|
|
1986
1993
|
this.home = this.createActionWrapper('AndroidHomeButton');
|
|
@@ -2009,7 +2016,7 @@ function mcp_tools_define_property(obj, key, value) {
|
|
|
2009
2016
|
return obj;
|
|
2010
2017
|
}
|
|
2011
2018
|
const debug = (0, logger_.getDebug)('mcp:android-tools');
|
|
2012
|
-
class AndroidMidsceneTools extends
|
|
2019
|
+
class AndroidMidsceneTools extends BaseMidsceneTools {
|
|
2013
2020
|
getCliReportSessionName() {
|
|
2014
2021
|
return 'midscene-android';
|
|
2015
2022
|
}
|
|
@@ -2087,8 +2094,8 @@ class AndroidMidsceneTools extends __rspack_external__midscene_shared_mcp_base_t
|
|
|
2087
2094
|
super(...args), mcp_tools_define_property(this, "initArgSpec", {
|
|
2088
2095
|
namespace: 'android',
|
|
2089
2096
|
shape: {
|
|
2090
|
-
deviceId:
|
|
2091
|
-
useScrcpy:
|
|
2097
|
+
deviceId: z.string().optional().describe('Android device ID (from adb devices)'),
|
|
2098
|
+
useScrcpy: z.boolean().optional().describe('Enable scrcpy accelerated screenshots')
|
|
2092
2099
|
},
|
|
2093
2100
|
cli: {
|
|
2094
2101
|
preferBareKeys: true
|
|
@@ -2100,5 +2107,4 @@ class AndroidMidsceneTools extends __rspack_external__midscene_shared_mcp_base_t
|
|
|
2100
2107
|
});
|
|
2101
2108
|
}
|
|
2102
2109
|
}
|
|
2103
|
-
|
|
2104
|
-
export { AndroidAgent, AndroidDevice, AndroidMidsceneTools, ScrcpyDeviceAdapter, agentFromAdbDevice, getConnectedDevices, getConnectedDevicesWithDetails, __webpack_exports__overrideAIConfig as overrideAIConfig };
|
|
2110
|
+
export { AndroidAgent, AndroidDevice, AndroidMidsceneTools, ScrcpyDeviceAdapter, agentFromAdbDevice, getConnectedDevices, getConnectedDevicesWithDetails, overrideAIConfig };
|