@aiscene/android 1.6.6 → 1.6.8

package/dist/lib/mcp-server.js

@@ -564,6 +564,7 @@ var __webpack_exports__ = {};
     };
     const external_node_assert_namespaceObject = require("node:assert");
     var external_node_assert_default = /*#__PURE__*/ __webpack_require__.n(external_node_assert_namespaceObject);
+    const external_node_child_process_namespaceObject = require("node:child_process");
     var external_node_fs_ = __webpack_require__("node:fs");
     var external_node_fs_default = /*#__PURE__*/ __webpack_require__.n(external_node_fs_);
     var external_node_module_ = __webpack_require__("node:module");
@@ -705,6 +706,9 @@ var __webpack_exports__ = {};
     const IME_STRATEGY_ALWAYS_YADB = 'always-yadb';
     const IME_STRATEGY_YADB_FOR_NON_ASCII = 'yadb-for-non-ascii';
     const debugDevice = (0, logger_.getDebug)('android:device');
+    function escapeForShell(text) {
+        return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
+    }
     class AndroidDevice {
         actionSpace() {
             const defaultActions = [
@@ -732,6 +736,12 @@ var __webpack_exports__ = {};
                         ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - clear the field without inputting new text.')),
                         locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
                     }),
+                    sample: {
+                        value: 'test@example.com',
+                        locate: {
+                            prompt: 'the email input field'
+                        }
+                    },
                     call: async (param)=>{
                         const element = param.locate;
                         if ('typeOnly' !== param.mode) await this.clearInput(element);
@@ -777,9 +787,21 @@ var __webpack_exports__ = {};
                         y: to.center[1]
                     });
                 }),
+                (0, device_namespaceObject.defineActionSwipe)(async (param)=>{
+                    const { startPoint, endPoint, duration, repeatCount } = (0, device_namespaceObject.normalizeMobileSwipeParam)(param, await this.size());
+                    for(let i = 0; i < repeatCount; i++)await this.mouseDrag(startPoint, endPoint, duration);
+                }),
                 (0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
                     await this.keyboardPress(param.keyName);
                 }),
+                (0, device_namespaceObject.defineActionCursorMove)(async (param)=>{
+                    const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
+                    const times = param.times ?? 1;
+                    for(let i = 0; i < times; i++){
+                        await this.keyboardPress(arrowKey);
+                        await (0, core_utils_namespaceObject.sleep)(100);
+                    }
+                }),
                 (0, device_namespaceObject.defineAction)({
                     name: 'LongPress',
                     description: 'Trigger a long press on the screen at specified element',
@@ -787,6 +809,11 @@ var __webpack_exports__ = {};
                         duration: core_namespaceObject.z.number().optional().describe('The duration of the long press in milliseconds'),
                         locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The element to be long pressed')
                     }),
+                    sample: {
+                        locate: {
+                            prompt: 'the message bubble'
+                        }
+                    },
                     call: async (param)=>{
                         const element = param.locate;
                         if (!element) throw new Error('LongPress requires an element to be located');
@@ -806,6 +833,12 @@ var __webpack_exports__ = {};
                         duration: core_namespaceObject.z.number().optional().describe('The duration of the pull (in milliseconds)'),
                         locate: (0, core_namespaceObject.getMidsceneLocationSchema)().optional().describe('The element to start the pull from (optional)')
                     }),
+                    sample: {
+                        direction: 'down',
+                        locate: {
+                            prompt: 'the center of the content list area'
+                        }
+                    },
                     call: async (param)=>{
                         const element = param.locate;
                         const startPoint = element ? {
@@ -818,6 +851,16 @@ var __webpack_exports__ = {};
                         else throw new Error(`Unknown pull direction: ${param.direction}`);
                     }
                 }),
+                (0, device_namespaceObject.defineActionPinch)(async (param)=>{
+                    const { centerX, centerY, startDistance, endDistance, duration } = (0, device_namespaceObject.normalizePinchParam)(param, await this.size());
+                    const { x: adjCenterX, y: adjCenterY } = await this.adjustCoordinates(centerX, centerY);
+                    const ratio = 0 !== adjCenterX && 0 !== centerX ? adjCenterX / centerX : 1;
+                    const adjStartDist = Math.round(startDistance * ratio);
+                    const adjEndDist = Math.round(endDistance * ratio);
+                    await this.ensureYadb();
+                    const adb = await this.getAdb();
+                    await adb.shell(`app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -pinch ${adjCenterX} ${adjCenterY} ${adjStartDist} ${adjEndDist} ${duration}`);
+                }),
                 (0, device_namespaceObject.defineActionClearInput)(async (param)=>{
                     await this.clearInput(param.locate);
                 })
@@ -962,61 +1005,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         async execYadb(keyboardContent) {
             await this.ensureYadb();
             const adb = await this.getAdb();
-            try {
-                const command = `app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "${keyboardContent}"`;
-                debugDevice(`Executing YADB input: "${keyboardContent}"`);
-                await adb.shell(command);
-                debugDevice(`YADB input completed: "${keyboardContent}"`);
-            } catch (error) {
-                const isAccessibilityConflict = error?.cause?.stderr?.includes('UiAutomationService') || error?.cause?.stderr?.includes('already registered') || error?.message?.includes('UiAutomationService');
-                if (isAccessibilityConflict) {
-                    debugDevice("YADB failed due to AccessibilityService conflict (likely Appium running), falling back to clipboard method");
-                    await this.inputViaClipboard(keyboardContent);
-                } else debugDevice(`YADB execution may have completed despite error: ${error}`);
-            }
-        }
-        async inputViaClipboard(text) {
-            const adb = await this.getAdb();
-            try {
-                debugDevice(`Inputting via clipboard: "${text}"`);
-                const escapedText = text.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-                const setClipboardCmd = `
-        content insert --uri content://settings/system --bind name:s:clipboard_text --bind value:s:"${escapedText}"
-      `;
-                await adb.shell(setClipboardCmd);
-                await (0, core_utils_namespaceObject.sleep)(100);
-                await adb.shell('input keyevent KEYCODE_PASTE');
-                await (0, core_utils_namespaceObject.sleep)(100);
-                debugDevice(`Clipboard input completed via content provider: "${text}"`);
-            } catch (error1) {
-                debugDevice(`Content provider clipboard failed, trying clipper app: ${error1}`);
-                try {
-                    const base64Text = Buffer.from(text, 'utf-8').toString('base64');
-                    await adb.shell(`am broadcast -a clipper.set -e text "${base64Text}"`);
-                    await (0, core_utils_namespaceObject.sleep)(100);
-                    await adb.shell('input keyevent KEYCODE_PASTE');
-                    await (0, core_utils_namespaceObject.sleep)(100);
-                    debugDevice(`Clipboard input completed via clipper: "${text}"`);
-                } catch (error2) {
-                    debugDevice(`All clipboard methods failed: ${error2}`);
-                    const isPureAscii = /^[\x00-\x7F]*$/.test(text);
-                    if (isPureAscii) {
-                        debugDevice(`Using ADB inputText for ASCII text: "${text}"`);
-                        await adb.inputText(text);
-                    } else await this.inputCharByChar(text);
-                }
-            }
-        }
-        async inputCharByChar(text) {
-            const adb = await this.getAdb();
-            debugDevice(`Inputting character by character (slow method): "${text}"`);
-            const chars = Array.from(text);
-            for (const char of chars){
-                if (' ' === char) await adb.shell('input keyevent KEYCODE_SPACE');
-                else await adb.shell(`input text "${char}"`);
-                await (0, core_utils_namespaceObject.sleep)(50);
-            }
-            debugDevice("Character-by-character input completed");
+            await adb.shell(`app_process${this.getDisplayArg()} -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard '${keyboardContent}'`);
         }
         async getElementsInfo() {
             return [];
@@ -1029,6 +1018,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         }
         async getScreenSize() {
             const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
+            debugDevice(`getScreenSize: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedSize=${!!this.cachedScreenSize}`);
             if (shouldCache && this.cachedScreenSize) return this.cachedScreenSize;
             const adb = await this.getAdb();
             if ('number' == typeof this.options?.displayId) try {
@@ -1162,6 +1152,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         }
         async getDisplayOrientation() {
             const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
+            debugDevice(`getDisplayOrientation: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedOrientation=${null !== this.cachedOrientation}`);
             if (shouldCache && null !== this.cachedOrientation) return this.cachedOrientation;
             const adb = await this.getAdb();
             let orientation = 0;
@@ -1187,6 +1178,15 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             if (shouldCache) this.cachedOrientation = orientation;
             return orientation;
         }
+        async getOrientedPhysicalSize() {
+            const info = await this.getDevicePhysicalInfo();
+            const isLandscape = 1 === info.orientation || 3 === info.orientation;
+            const shouldSwap = true !== info.isCurrentOrientation && isLandscape;
+            return {
+                width: shouldSwap ? info.physicalHeight : info.physicalWidth,
+                height: shouldSwap ? info.physicalWidth : info.physicalHeight
+            };
+        }
         async size() {
             const deviceInfo = await this.getDevicePhysicalInfo();
             const adapter = this.getScrcpyAdapter();
@@ -1213,7 +1213,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                 height: logicalHeight
             };
         }
-        async cacheFeatureForPoint(center, options) {
+        async cacheFeatureForPoint(center) {
             const { width, height } = await this.size();
             debugDevice('cacheFeatureForPoint: center=[%s,%s], screen=[%s,%s]', center[0], center[1], width, height);
             return {
@@ -1293,14 +1293,23 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             const androidScreenshotPath = `/data/local/tmp/ms_${screenshotId}.png`;
             const useShellScreencap = 'number' == typeof this.options?.displayId;
             try {
-                if (useShellScreencap) throw new Error('Using shell screencap for displayId');
-                debugDevice('Taking screenshot via adb.takeScreenshot');
-                screenshotBuffer = await adb.takeScreenshot(null);
-                debugDevice('adb.takeScreenshot completed');
-                if (!screenshotBuffer) throw new Error('Failed to capture screenshot: screenshotBuffer is null');
-                if (!(0, img_namespaceObject.isValidPNGImageBuffer)(screenshotBuffer)) {
-                    debugDevice('Invalid image buffer detected: not a valid image format');
-                    throw new Error('Screenshot buffer has invalid format: could not find valid image signature');
+                if (!useShellScreencap && this.takeScreenshotFailCount < AndroidDevice.TAKE_SCREENSHOT_FAIL_THRESHOLD) {
+                    debugDevice('Taking screenshot via adb.takeScreenshot');
+                    screenshotBuffer = await adb.takeScreenshot(null);
+                    debugDevice('adb.takeScreenshot completed');
+                    if (!screenshotBuffer) {
+                        this.takeScreenshotFailCount++;
+                        throw new Error('Failed to capture screenshot: screenshotBuffer is null');
+                    }
+                    if (!(0, img_namespaceObject.isValidImageBuffer)(screenshotBuffer)) {
+                        debugDevice('Invalid image buffer detected: not a valid image format');
+                        this.takeScreenshotFailCount++;
+                        throw new Error('Screenshot buffer has invalid format: could not find valid image signature');
+                    }
+                    this.takeScreenshotFailCount = 0;
+                } else {
+                    if (this.takeScreenshotFailCount >= AndroidDevice.TAKE_SCREENSHOT_FAIL_THRESHOLD) debugDevice('Skipping takeScreenshot (failed %d consecutive times), using shell screencap directly', this.takeScreenshotFailCount);
+                    throw new Error('Using shell screencap directly');
                 }
                 const validScreenshotBufferSize = this.options?.minScreenshotBufferSize ?? 10240;
                 if (validScreenshotBufferSize > 0 && screenshotBuffer.length < validScreenshotBufferSize) {
@@ -1329,12 +1338,21 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                     screenshotBuffer = await external_node_fs_default().promises.readFile(screenshotPath);
                     const validScreenshotBufferSize = this.options?.minScreenshotBufferSize ?? 10240;
                     if (!screenshotBuffer || validScreenshotBufferSize > 0 && screenshotBuffer.length < validScreenshotBufferSize) throw new Error(`Fallback screenshot validation failed: buffer size ${screenshotBuffer?.length || 0} bytes (minimum: ${validScreenshotBufferSize})`);
-                    if (!(0, img_namespaceObject.isValidPNGImageBuffer)(screenshotBuffer)) throw new Error('Fallback screenshot buffer has invalid PNG format');
+                    if (!(0, img_namespaceObject.isValidImageBuffer)(screenshotBuffer)) throw new Error('Fallback screenshot buffer has invalid PNG format');
                     debugDevice(`Fallback screenshot validated successfully: ${screenshotBuffer.length} bytes`);
                 } finally{
-                    Promise.resolve().then(()=>adb.shell(`rm ${androidScreenshotPath}`)).catch((error)=>{
-                        debugDevice(`Failed to delete remote screenshot: ${error}`);
+                    const adbPath = adb.executable?.path ?? 'adb';
+                    const child = (0, external_node_child_process_namespaceObject.execFile)(adbPath, [
+                        '-s',
+                        this.deviceId,
+                        'shell',
+                        `rm ${androidScreenshotPath}`
+                    ], {
+                        timeout: 3000
+                    }, (err)=>{
+                        if (err) debugDevice('Failed to delete remote screenshot: %s', err.message);
                     });
+                    child.unref();
                 }
             }
             if (!screenshotBuffer) throw new Error('Failed to capture screenshot: all methods failed');
@@ -1497,24 +1515,32 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             if (!this.yadbPushed) {
                 const adb = await this.getAdb();
                 const androidPkgJson = (0, external_node_module_.createRequire)(__rslib_import_meta_url__).resolve('@aiscene/android/package.json');
-                const yadbDir = external_node_path_default().join(external_node_path_default().dirname(androidPkgJson), 'bin');
-                await adb.push(yadbDir, '/data/local/tmp');
+                const yadbBin = external_node_path_default().join(external_node_path_default().dirname(androidPkgJson), 'bin', 'yadb');
+                await adb.push(yadbBin, '/data/local/tmp');
                 this.yadbPushed = true;
             }
         }
         shouldUseYadbForText(text) {
             const hasNonAscii = /[\x80-\uFFFF]/.test(text);
             const hasFormatSpecifiers = /%[a-zA-Z]/.test(text);
-            return hasNonAscii || hasFormatSpecifiers;
+            const hasShellSpecialChars = /[\\`$]/.test(text);
+            const hasBothQuotes = text.includes('"') && text.includes("'");
+            return hasNonAscii || hasFormatSpecifiers || hasShellSpecialChars || hasBothQuotes;
         }
         async keyboardType(text, options) {
             if (!text) return;
             const adb = await this.getAdb();
-            const shouldUseYadb = this.shouldUseYadbForText(text);
             const IME_STRATEGY = (this.options?.imeStrategy || env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.MIDSCENE_ANDROID_IME_STRATEGY)) ?? IME_STRATEGY_YADB_FOR_NON_ASCII;
             const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
-            if (IME_STRATEGY === IME_STRATEGY_ALWAYS_YADB || IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII && shouldUseYadb) await this.execYadb(text);
-            else await adb.inputText(text);
+            const useYadb = IME_STRATEGY === IME_STRATEGY_ALWAYS_YADB || IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII && this.shouldUseYadbForText(text);
+            if (useYadb) await this.execYadb(escapeForShell(text));
+            else {
+                const segments = text.split('\n');
+                for(let i = 0; i < segments.length; i++){
+                    if (segments[i].length > 0) await adb.inputText(segments[i]);
+                    if (i < segments.length - 1) await adb.keyevent(66);
+                }
+            }
             if (true === shouldAutoDismissKeyboard) await this.hideKeyboard(options);
         }
         normalizeKeyName(key) {
@@ -1562,12 +1588,12 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         }
         async mouseClick(x, y) {
             const adb = await this.getAdb();
-            const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
+            const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
             await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} 150`);
         }
         async mouseDoubleClick(x, y) {
             const adb = await this.getAdb();
-            const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
+            const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
             const tapCommand = `input${this.getDisplayArg()} tap ${adjustedX} ${adjustedY}`;
             await adb.shell(tapCommand);
             await (0, core_utils_namespaceObject.sleep)(50);
@@ -1578,8 +1604,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         }
         async mouseDrag(from, to, duration) {
             const adb = await this.getAdb();
-            const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
-            const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
+            const { x: fromX, y: fromY } = await this.adjustCoordinates(from.x, from.y);
+            const { x: toX, y: toY } = await this.adjustCoordinates(to.x, to.y);
             const swipeDuration = duration ?? defaultNormalScrollDuration;
             await adb.shell(`input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`);
         }
@@ -1589,16 +1615,16 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             const n = 4;
             const startX = Math.round(deltaX < 0 ? width / n * (n - 1) : width / n);
             const startY = Math.round(deltaY < 0 ? height / n * (n - 1) : height / n);
-            const maxNegativeDeltaX = startX;
-            const maxPositiveDeltaX = Math.round(width / n * (n - 1));
-            const maxNegativeDeltaY = startY;
-            const maxPositiveDeltaY = Math.round(height / n * (n - 1));
+            const maxPositiveDeltaX = startX;
+            const maxNegativeDeltaX = width - startX;
+            const maxPositiveDeltaY = startY;
+            const maxNegativeDeltaY = height - startY;
             deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
             deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
             const endX = Math.round(startX - deltaX);
             const endY = Math.round(startY - deltaY);
-            const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(startX, startY);
-            const { x: adjustedEndX, y: adjustedEndY } = this.adjustCoordinates(endX, endY);
+            const { x: adjustedStartX, y: adjustedStartY } = await this.adjustCoordinates(startX, startY);
+            const { x: adjustedEndX, y: adjustedEndY } = await this.adjustCoordinates(endX, endY);
             const adb = await this.getAdb();
             const swipeDuration = duration ?? defaultNormalScrollDuration;
             await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`);
@@ -1609,6 +1635,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             this.cachedPhysicalDisplayId = void 0;
             this.cachedScreenSize = null;
             this.cachedOrientation = null;
+            this.scalingRatio = 1;
             if (this.scrcpyAdapter) {
                 await this.scrcpyAdapter.disconnect();
                 this.scrcpyAdapter = null;
@@ -1648,7 +1675,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         }
         async longPress(x, y, duration = 2000) {
             const adb = await this.getAdb();
-            const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
+            const { x: adjustedX, y: adjustedY } = await this.adjustCoordinates(x, y);
             await adb.shell(`input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} ${duration}`);
         }
         async pullDown(startPoint, distance, duration = 800) {
@@ -1670,8 +1697,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         }
         async pullDrag(from, to, duration) {
             const adb = await this.getAdb();
-            const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
-            const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
+            const { x: fromX, y: fromY } = await this.adjustCoordinates(from.x, from.y);
+            const { x: toX, y: toY } = await this.adjustCoordinates(to.x, to.y);
             await adb.shell(`input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${duration}`);
         }
         async pullUp(startPoint, distance, duration = 600) {
@@ -1758,7 +1785,6 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             device_define_property(this, "yadbPushed", false);
             device_define_property(this, "devicePixelRatio", 1);
             device_define_property(this, "devicePixelRatioInitialized", false);
-            device_define_property(this, "scalingRatio", 1);
             device_define_property(this, "adb", null);
             device_define_property(this, "connectingAdb", null);
             device_define_property(this, "destroyed", false);
@@ -1769,6 +1795,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             device_define_property(this, "cachedPhysicalDisplayId", void 0);
             device_define_property(this, "scrcpyAdapter", null);
             device_define_property(this, "appNameMapping", {});
+            device_define_property(this, "scalingRatio", 1);
+            device_define_property(this, "takeScreenshotFailCount", 0);
             device_define_property(this, "interfaceType", 'android');
             device_define_property(this, "uri", void 0);
             device_define_property(this, "options", void 0);
@@ -1778,6 +1806,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             this.customActions = options?.customActions;
         }
     }
+    device_define_property(AndroidDevice, "TAKE_SCREENSHOT_FAIL_THRESHOLD", 3);
     const runAdbShellParamSchema = core_namespaceObject.z.object({
         command: core_namespaceObject.z.string().describe('ADB shell command to execute')
     });
@@ -1790,6 +1819,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                 description: 'Execute ADB shell command on Android device',
                 interfaceAlias: 'runAdbShell',
                 paramSchema: runAdbShellParamSchema,
+                sample: {
+                    command: 'dumpsys window displays | grep -E "mCurrentFocus"'
+                },
                 call: async (param)=>{
                     if (!param.command || '' === param.command.trim()) throw new Error('RunAdbShell requires a non-empty command parameter');
                     const adb = await device.getAdb();
@@ -1801,6 +1833,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                 description: 'Launch an Android app or URL',
                 interfaceAlias: 'launch',
                 paramSchema: launchParamSchema,
+                sample: {
+                    uri: 'com.example.app'
+                },
                 call: async (param)=>{
                     if (!param.uri || '' === param.uri.trim()) throw new Error('Launch requires a non-empty uri parameter');
                     await device.launch(param.uri);
@@ -1953,7 +1988,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         constructor(toolsManager){
             super({
                 name: '@midscene/android-mcp',
-                version: "1.6.6",
+                version: "1.6.8",
                 description: 'Control the Android device using natural language commands'
             }, toolsManager);
         }

package/dist/types/index.d.ts

@@ -10,13 +10,10 @@ import { AndroidDeviceOpt } from '@midscene/core/device';
 import { BaseMidsceneTools } from '@midscene/shared/mcp';
 import { Device } from 'appium-adb';
 import { DeviceAction } from '@midscene/core';
-import { ElementCacheFeature } from '@midscene/core';
 import type { ElementInfo } from '@midscene/shared/extractor';
-import { IModelConfig } from '@midscene/shared/env';
 import { InterfaceType } from '@midscene/core';
 import { overrideAIConfig } from '@midscene/shared/env';
 import { Point } from '@midscene/core';
-import { Rect } from '@midscene/core';
 import { Size } from '@midscene/core';
 import { ToolDefinition } from '@midscene/shared/mcp';
 
@@ -68,7 +65,6 @@ export declare class AndroidDevice implements AbstractInterface {
     private yadbPushed;
     private devicePixelRatio;
     private devicePixelRatioInitialized;
-    private scalingRatio;
     private adb;
     private connectingAdb;
     private destroyed;
@@ -79,6 +75,9 @@ export declare class AndroidDevice implements AbstractInterface {
     private cachedPhysicalDisplayId;
     private scrcpyAdapter;
     private appNameMapping;
+    private scalingRatio;
+    private takeScreenshotFailCount;
+    private static readonly TAKE_SCREENSHOT_FAIL_THRESHOLD;
     interfaceType: InterfaceType;
     uri: string | undefined;
     options?: AndroidDeviceOpt;
@@ -109,16 +108,6 @@ export declare class AndroidDevice implements AbstractInterface {
     private resolvePackageName;
     launch(uri: string): Promise<AndroidDevice>;
     execYadb(keyboardContent: string): Promise<void>;
-    /**
-     * 通过剪贴板输入文本（备用方案，当YADB不可用时）
-     * 支持中文等非ASCII字符
-     */
-    private inputViaClipboard;
-    /**
-     * 逐字符输入文本（最后备用方案）
-     * 通过模拟按键输入，支持中文（如果有对应输入法）
-     */
-    private inputCharByChar;
     getElementsInfo(): Promise<ElementInfo[]>;
     getElementsNodeTree(): Promise<any>;
     getScreenSize(): Promise<{
@@ -130,22 +119,38 @@ export declare class AndroidDevice implements AbstractInterface {
     private initializeDevicePixelRatio;
     getDisplayDensity(): Promise<number>;
     getDisplayOrientation(): Promise<number>;
-    size(): Promise<Size>;
     /**
-     * Generate cache feature for an element at given center point
-     * Cache is based on test case ID + device ID, so screen size is guaranteed to be consistent
-     * for same test case on same device. We store exact coordinates for maximum accuracy.
+     * Get physical screen dimensions adjusted for current orientation.
+     * Swaps width/height when the device is in landscape and the reported
+     * dimensions do not already reflect the current orientation.
      */
-    cacheFeatureForPoint(center: [number, number], options?: {
-        targetDescription?: string;
-        modelConfig?: IModelConfig;
-    }): Promise<ElementCacheFeature>;
+    private getOrientedPhysicalSize;
+    size(): Promise<Size>;
+    cacheFeatureForPoint(center: [number, number]): Promise<{
+        centerX: number;
+        centerY: number;
+        screenSize: {
+            width: number;
+            height: number;
+        };
+    }>;
+    rectMatchesCacheFeature(feature: {
+        centerX: number;
+        centerY: number;
+        screenSize: {
+            width: number;
+            height: number;
+        };
+    }): Promise<{
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+    }>;
     /**
-     * Find element rect from cache feature
-     * Since cache is keyed by test case ID + device ID, screen size will be identical
-     * We return the exact cached coordinates for maximum accuracy.
+     * Convert logical coordinates (from AI) back to physical coordinates (for ADB).
+     * The ratio is derived from size(), so overriding size() alone is sufficient.
      */
-    rectMatchesCacheFeature(feature: ElementCacheFeature): Promise<Rect>;
     private adjustCoordinates;
     /**
      * Calculate the end point for scroll operations based on start point, scroll delta, and screen boundaries.
@@ -174,11 +179,15 @@ export declare class AndroidDevice implements AbstractInterface {
     scrollRight(distance?: number, startPoint?: Point): Promise<void>;
     ensureYadb(): Promise<void>;
     /**
-     * Check if text contains characters that may cause issues with ADB inputText
-     * This includes:
-     * - Non-ASCII characters (Unicode characters like ö, é, ñ, Chinese, Japanese, etc.)
-     * - Format specifiers that may be interpreted by shell (%, $)
-     * - Special shell characters that need escaping
+     * Check if text contains characters that may cause issues with ADB inputText.
+     * appium-adb's inputText has known bugs with certain characters:
+     * - Backslash causes broken shell quoting
+     * - Backtick is not escaped at all
+     * - Text containing both " and ' throws an error
+     * - Dollar sign can cause variable expansion issues
+     *
+     * For these characters, we route through yadb which handles them correctly
+     * via escapeForShell + double-quoted shell context.
      */
     private shouldUseYadbForText;
     keyboardType(text: string, options?: AndroidDeviceInputOpt): Promise<void>;