@midscene/android 1.7.0 → 1.7.1-beta-20260408073050.0

package/dist/lib/index.js

@@ -612,6 +612,18 @@ var __webpack_exports__ = {};
     const defaultNormalScrollDuration = 1000;
     const IME_STRATEGY_ALWAYS_YADB = 'always-yadb';
     const IME_STRATEGY_YADB_FOR_NON_ASCII = 'yadb-for-non-ascii';
+    const MIDSCENE_IME_PACKAGE = 'com.midscene.ime';
+    const MIDSCENE_IME_ID = `${MIDSCENE_IME_PACKAGE}/.MidsceneIME`;
+    const MIDSCENE_IME_DISMISS_ACTION = `${MIDSCENE_IME_PACKAGE}.DISMISS`;
+    const AUTO_DISMISS_KEYBOARD_SCHEMA = core_namespaceObject.z.union([
+        core_namespaceObject.z.boolean(),
+        core_namespaceObject.z["enum"]([
+            'back',
+            'esc',
+            'midscene-ime',
+            'midscene-ime-auto-install'
+        ])
+    ]).optional().describe("Dismiss the keyboard after input. Use `true` for the default ESC-based behavior, `false` to leave the keyboard open, `'back'`/`'esc'` for explicit key-event strategies, or `'midscene-ime'` / `'midscene-ime-auto-install'` for the zero-key-event helper IME flow.");
     const debugDevice = (0, logger_.getDebug)('android:device');
     function escapeForShell(text) {
         return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
@@ -635,7 +647,7 @@ var __webpack_exports__ = {};
                     interfaceAlias: 'aiInput',
                     paramSchema: core_namespaceObject.z.object({
                         value: core_namespaceObject.z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-                        autoDismissKeyboard: core_namespaceObject.z.boolean().optional().describe('If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.'),
+                        autoDismissKeyboard: AUTO_DISMISS_KEYBOARD_SCHEMA,
                         mode: core_namespaceObject.z.preprocess((val)=>'append' === val ? 'typeOnly' : val, core_namespaceObject.z["enum"]([
                             'replace',
                             'clear',
@@ -881,6 +893,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         setAppNameMapping(mapping) {
             this.appNameMapping = mapping;
         }
+        setInstallApprovalHandler(handler) {
+            this.installApprovalHandler = handler;
+        }
         resolvePackageName(appName) {
             const normalizedAppName = (0, shared_utils_namespaceObject.normalizeForComparison)(appName);
             return this.appNameMapping[normalizedAppName];
@@ -1419,7 +1434,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             if (!text) return;
             const adb = await this.getAdb();
             const IME_STRATEGY = (this.options?.imeStrategy || env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.MIDSCENE_ANDROID_IME_STRATEGY)) ?? IME_STRATEGY_YADB_FOR_NON_ASCII;
-            const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
+            const autoDismiss = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? 'esc';
+            const shouldAutoDismissKeyboard = false !== autoDismiss;
             const useYadb = IME_STRATEGY === IME_STRATEGY_ALWAYS_YADB || IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII && this.shouldUseYadbForText(text);
             if (useYadb) await this.execYadb(escapeForShell(text), {
                 overwrite: options?.overwrite
@@ -1537,6 +1553,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             }
             this.connectingAdb = null;
             this.yadbPushed = false;
+            this.midsceneImeInstalled = false;
         }
         async getTimestamp() {
             const adb = await this.getAdb();
@@ -1636,43 +1653,135 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                 return null;
             }
         }
+        resolveAutoDismissKeyboard(options) {
+            const raw = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
+            if (true === raw) ;
+            else if (false === raw) return false;
+            else if (void 0 !== raw) return raw;
+            const legacy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy;
+            if ('back-first' === legacy) return 'back';
+            return 'esc';
+        }
+        async waitForKeyboardHidden(adb, timeoutMs) {
+            const startTime = Date.now();
+            const intervalMs = 100;
+            while(Date.now() - startTime < timeoutMs){
+                await (0, utils_namespaceObject.sleep)(intervalMs);
+                const currentStatus = await adb.isSoftKeyboardPresent();
+                const isStillShown = 'boolean' == typeof currentStatus ? currentStatus : currentStatus?.isKeyboardShown;
+                if (!isStillShown) return true;
+            }
+            return false;
+        }
+        async ensureMidsceneImeInstalled(adb, autoApprove = false) {
+            if (this.midsceneImeInstalled) return;
+            try {
+                const output = await adb.shell(`pm list packages ${MIDSCENE_IME_PACKAGE}`);
+                if (output.includes(MIDSCENE_IME_PACKAGE)) {
+                    await adb.shell(`ime enable ${MIDSCENE_IME_ID}`);
+                    this.midsceneImeInstalled = true;
+                    return;
+                }
+            } catch  {}
+            const androidPkgJson = (0, external_node_module_.createRequire)(__rslib_import_meta_url__).resolve('@midscene/android/package.json');
+            const apkPath = external_node_path_default().join(external_node_path_default().dirname(androidPkgJson), 'bin', 'midscene-ime.apk');
+            if (!external_node_fs_default().existsSync(apkPath)) throw new Error(`MidsceneIME APK not found at ${apkPath}. Run the @midscene/android prepack/build step to generate bin/midscene-ime.apk before using the MidsceneIME keyboard-dismiss strategy.`);
+            try {
+                await adb.install(apkPath);
+                await adb.shell(`ime enable ${MIDSCENE_IME_ID}`);
+                this.midsceneImeInstalled = true;
+                return;
+            } catch (e) {
+                const errorMsg = String(e);
+                if (!errorMsg.includes('INSTALL_FAILED_USER_RESTRICTED')) throw e;
+                if (!autoApprove || !this.installApprovalHandler) throw new Error(`APK installation was blocked by the device (INSTALL_FAILED_USER_RESTRICTED). On some ROMs (e.g. MIUI), enable "Install via USB" in Developer Options, or use autoDismissKeyboard: 'midscene-ime-auto-install' with an agent to auto-approve. Original error: ${e}`);
+                debugDevice('Install blocked by device, retrying with approval handler...');
+            }
+            await adb.push(apkPath, '/data/local/tmp/midscene-ime.apk');
+            const installPromise = adb.shell('pm install -r -t /data/local/tmp/midscene-ime.apk');
+            await (0, utils_namespaceObject.sleep)(2000);
+            try {
+                await this.installApprovalHandler();
+            } catch (approvalError) {
+                debugDevice(`Install approval handler failed: ${approvalError}`);
+            }
+            try {
+                const result = await installPromise;
+                if (result.includes('Success')) {
+                    await adb.shell(`ime enable ${MIDSCENE_IME_ID}`);
+                    this.midsceneImeInstalled = true;
+                    debugDevice('MidsceneIME installed via auto-approval');
+                    return;
+                }
+                throw new Error(`Install failed: ${result}`);
+            } catch (e) {
+                throw new Error(`Failed to install MidsceneIME even with auto-approval. Please manually enable "Install via USB" in Developer Options. Original error: ${e}`);
+            }
+        }
+        async hideKeyboardViaMidsceneIme(adb, timeoutMs) {
+            const originalIme = await adb.shell('settings get secure default_input_method');
+            const trimmedOriginalIme = originalIme.trim();
+            try {
+                await adb.shell(`ime set ${MIDSCENE_IME_ID}`);
+                await (0, utils_namespaceObject.sleep)(500);
+                await adb.shell(`am broadcast -a ${MIDSCENE_IME_DISMISS_ACTION}`);
+                await (0, utils_namespaceObject.sleep)(500);
+                if (trimmedOriginalIme && trimmedOriginalIme !== MIDSCENE_IME_ID) await adb.shell(`ime set ${trimmedOriginalIme}`);
+                if (await this.waitForKeyboardHidden(adb, timeoutMs)) {
+                    debugDevice('Keyboard hidden successfully with MidsceneIME');
+                    return true;
+                }
+                debugDevice('MidsceneIME did not dismiss keyboard');
+                return false;
+            } catch (e) {
+                if (trimmedOriginalIme && trimmedOriginalIme !== MIDSCENE_IME_ID) try {
+                    await adb.shell(`ime set ${trimmedOriginalIme}`);
+                } catch  {}
+                throw e;
+            }
+        }
         async hideKeyboard(options, timeoutMs = 1000) {
             const adb = await this.getAdb();
-            const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
+            const strategy = this.resolveAutoDismissKeyboard(options);
+            if (false === strategy) return false;
             const keyboardStatus = await adb.isSoftKeyboardPresent();
             const isKeyboardShown = 'boolean' == typeof keyboardStatus ? keyboardStatus : keyboardStatus?.isKeyboardShown;
             if (!isKeyboardShown) {
                 debugDevice('Keyboard has no UI; no closing necessary');
                 return false;
             }
-            const keyCodes = 'back-first' === keyboardDismissStrategy ? [
-                4,
-                111
-            ] : [
-                111,
-                4
-            ];
-            for (const keyCode of keyCodes){
-                await adb.keyevent(keyCode);
-                const startTime = Date.now();
-                const intervalMs = 100;
-                while(Date.now() - startTime < timeoutMs){
-                    await (0, utils_namespaceObject.sleep)(intervalMs);
-                    const currentStatus = await adb.isSoftKeyboardPresent();
-                    const isStillShown = 'boolean' == typeof currentStatus ? currentStatus : currentStatus?.isKeyboardShown;
-                    if (!isStillShown) {
-                        debugDevice(`Keyboard hidden successfully with keycode ${keyCode}`);
-                        return true;
-                    }
+            if ('midscene-ime' === strategy || 'midscene-ime-auto-install' === strategy) {
+                const autoApprove = 'midscene-ime-auto-install' === strategy;
+                try {
+                    await this.ensureMidsceneImeInstalled(adb, autoApprove);
+                    const hidden = await this.hideKeyboardViaMidsceneIme(adb, timeoutMs);
+                    if (!hidden) console.warn('Warning: MidsceneIME was invoked but the software keyboard is still visible');
+                    return hidden;
+                } catch (e) {
+                    const installHint = autoApprove ? 'Failed to use MidsceneIME auto-install for keyboard dismissal. MidsceneIME will not fall back to ESC/BACK because that can clear user input. Please enable USB installs on the device or preinstall the helper APK manually: adb install <path-to-midscene-ime.apk>.' : "Failed to use MidsceneIME for keyboard dismissal. Please install the APK manually: adb install <path-to-midscene-ime.apk>, or use autoDismissKeyboard: 'midscene-ime-auto-install' with an AndroidAgent to auto-approve installation prompts.";
+                    throw new Error(`${installHint} Original error: ${e}`);
                 }
-                debugDevice(`Keyboard still shown after keycode ${keyCode}, trying next key`);
             }
-            console.warn('Warning: Failed to hide the software keyboard after trying both ESC and BACK keys');
+            const keyCode = 'back' === strategy ? 4 : 111;
+            await adb.keyevent(keyCode);
+            if (await this.waitForKeyboardHidden(adb, timeoutMs)) {
+                debugDevice(`Keyboard hidden successfully with keycode ${keyCode} (${strategy})`);
+                return true;
+            }
+            const fallbackKeyCode = 'back' === strategy ? 111 : 4;
+            await adb.keyevent(fallbackKeyCode);
+            if (await this.waitForKeyboardHidden(adb, timeoutMs)) {
+                debugDevice(`Keyboard hidden successfully with fallback keycode ${fallbackKeyCode}`);
+                return true;
+            }
+            console.warn('Warning: Failed to hide the software keyboard after trying all strategies');
             return false;
         }
         constructor(deviceId, options){
             device_define_property(this, "deviceId", void 0);
             device_define_property(this, "yadbPushed", false);
+            device_define_property(this, "midsceneImeInstalled", false);
+            device_define_property(this, "installApprovalHandler", void 0);
             device_define_property(this, "devicePixelRatio", 1);
             device_define_property(this, "devicePixelRatioInitialized", false);
             device_define_property(this, "adb", null);
@@ -1984,6 +2093,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             super(device, opts), agent_define_property(this, "back", void 0), agent_define_property(this, "home", void 0), agent_define_property(this, "recentApps", void 0), agent_define_property(this, "appNameMapping", void 0);
             this.appNameMapping = (0, shared_utils_namespaceObject.mergeAndNormalizeAppNameMapping)(defaultAppNameMapping, opts?.appNameMapping);
             device.setAppNameMapping(this.appNameMapping);
+            device.setInstallApprovalHandler(async ()=>{
+                await this.aiAct('A system dialog is asking to approve USB installation. Click the "Continue Install", "Allow", "Install" or similar approval button.');
+            });
             this.back = this.createActionWrapper('AndroidBackButton');
             this.home = this.createActionWrapper('AndroidHomeButton');
             this.recentApps = this.createActionWrapper('AndroidRecentAppsButton');

package/dist/lib/mcp-server.js

@@ -705,6 +705,18 @@ var __webpack_exports__ = {};
     const defaultNormalScrollDuration = 1000;
     const IME_STRATEGY_ALWAYS_YADB = 'always-yadb';
     const IME_STRATEGY_YADB_FOR_NON_ASCII = 'yadb-for-non-ascii';
+    const MIDSCENE_IME_PACKAGE = 'com.midscene.ime';
+    const MIDSCENE_IME_ID = `${MIDSCENE_IME_PACKAGE}/.MidsceneIME`;
+    const MIDSCENE_IME_DISMISS_ACTION = `${MIDSCENE_IME_PACKAGE}.DISMISS`;
+    const AUTO_DISMISS_KEYBOARD_SCHEMA = core_namespaceObject.z.union([
+        core_namespaceObject.z.boolean(),
+        core_namespaceObject.z["enum"]([
+            'back',
+            'esc',
+            'midscene-ime',
+            'midscene-ime-auto-install'
+        ])
+    ]).optional().describe("Dismiss the keyboard after input. Use `true` for the default ESC-based behavior, `false` to leave the keyboard open, `'back'`/`'esc'` for explicit key-event strategies, or `'midscene-ime'` / `'midscene-ime-auto-install'` for the zero-key-event helper IME flow.");
     const debugDevice = (0, logger_.getDebug)('android:device');
     function escapeForShell(text) {
         return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
@@ -728,7 +740,7 @@ var __webpack_exports__ = {};
                     interfaceAlias: 'aiInput',
                     paramSchema: core_namespaceObject.z.object({
                         value: core_namespaceObject.z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-                        autoDismissKeyboard: core_namespaceObject.z.boolean().optional().describe('If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.'),
+                        autoDismissKeyboard: AUTO_DISMISS_KEYBOARD_SCHEMA,
                         mode: core_namespaceObject.z.preprocess((val)=>'append' === val ? 'typeOnly' : val, core_namespaceObject.z["enum"]([
                             'replace',
                             'clear',
@@ -974,6 +986,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         setAppNameMapping(mapping) {
             this.appNameMapping = mapping;
         }
+        setInstallApprovalHandler(handler) {
+            this.installApprovalHandler = handler;
+        }
         resolvePackageName(appName) {
             const normalizedAppName = (0, utils_namespaceObject.normalizeForComparison)(appName);
             return this.appNameMapping[normalizedAppName];
@@ -1512,7 +1527,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             if (!text) return;
             const adb = await this.getAdb();
             const IME_STRATEGY = (this.options?.imeStrategy || env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.MIDSCENE_ANDROID_IME_STRATEGY)) ?? IME_STRATEGY_YADB_FOR_NON_ASCII;
-            const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
+            const autoDismiss = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? 'esc';
+            const shouldAutoDismissKeyboard = false !== autoDismiss;
             const useYadb = IME_STRATEGY === IME_STRATEGY_ALWAYS_YADB || IME_STRATEGY === IME_STRATEGY_YADB_FOR_NON_ASCII && this.shouldUseYadbForText(text);
             if (useYadb) await this.execYadb(escapeForShell(text), {
                 overwrite: options?.overwrite
@@ -1630,6 +1646,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             }
             this.connectingAdb = null;
             this.yadbPushed = false;
+            this.midsceneImeInstalled = false;
         }
         async getTimestamp() {
             const adb = await this.getAdb();
@@ -1729,43 +1746,135 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                 return null;
             }
         }
+        resolveAutoDismissKeyboard(options) {
+            const raw = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
+            if (true === raw) ;
+            else if (false === raw) return false;
+            else if (void 0 !== raw) return raw;
+            const legacy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy;
+            if ('back-first' === legacy) return 'back';
+            return 'esc';
+        }
+        async waitForKeyboardHidden(adb, timeoutMs) {
+            const startTime = Date.now();
+            const intervalMs = 100;
+            while(Date.now() - startTime < timeoutMs){
+                await (0, core_utils_namespaceObject.sleep)(intervalMs);
+                const currentStatus = await adb.isSoftKeyboardPresent();
+                const isStillShown = 'boolean' == typeof currentStatus ? currentStatus : currentStatus?.isKeyboardShown;
+                if (!isStillShown) return true;
+            }
+            return false;
+        }
+        async ensureMidsceneImeInstalled(adb, autoApprove = false) {
+            if (this.midsceneImeInstalled) return;
+            try {
+                const output = await adb.shell(`pm list packages ${MIDSCENE_IME_PACKAGE}`);
+                if (output.includes(MIDSCENE_IME_PACKAGE)) {
+                    await adb.shell(`ime enable ${MIDSCENE_IME_ID}`);
+                    this.midsceneImeInstalled = true;
+                    return;
+                }
+            } catch  {}
+            const androidPkgJson = (0, external_node_module_.createRequire)(__rslib_import_meta_url__).resolve('@midscene/android/package.json');
+            const apkPath = external_node_path_default().join(external_node_path_default().dirname(androidPkgJson), 'bin', 'midscene-ime.apk');
+            if (!external_node_fs_default().existsSync(apkPath)) throw new Error(`MidsceneIME APK not found at ${apkPath}. Run the @midscene/android prepack/build step to generate bin/midscene-ime.apk before using the MidsceneIME keyboard-dismiss strategy.`);
+            try {
+                await adb.install(apkPath);
+                await adb.shell(`ime enable ${MIDSCENE_IME_ID}`);
+                this.midsceneImeInstalled = true;
+                return;
+            } catch (e) {
+                const errorMsg = String(e);
+                if (!errorMsg.includes('INSTALL_FAILED_USER_RESTRICTED')) throw e;
+                if (!autoApprove || !this.installApprovalHandler) throw new Error(`APK installation was blocked by the device (INSTALL_FAILED_USER_RESTRICTED). On some ROMs (e.g. MIUI), enable "Install via USB" in Developer Options, or use autoDismissKeyboard: 'midscene-ime-auto-install' with an agent to auto-approve. Original error: ${e}`);
+                debugDevice('Install blocked by device, retrying with approval handler...');
+            }
+            await adb.push(apkPath, '/data/local/tmp/midscene-ime.apk');
+            const installPromise = adb.shell('pm install -r -t /data/local/tmp/midscene-ime.apk');
+            await (0, core_utils_namespaceObject.sleep)(2000);
+            try {
+                await this.installApprovalHandler();
+            } catch (approvalError) {
+                debugDevice(`Install approval handler failed: ${approvalError}`);
+            }
+            try {
+                const result = await installPromise;
+                if (result.includes('Success')) {
+                    await adb.shell(`ime enable ${MIDSCENE_IME_ID}`);
+                    this.midsceneImeInstalled = true;
+                    debugDevice('MidsceneIME installed via auto-approval');
+                    return;
+                }
+                throw new Error(`Install failed: ${result}`);
+            } catch (e) {
+                throw new Error(`Failed to install MidsceneIME even with auto-approval. Please manually enable "Install via USB" in Developer Options. Original error: ${e}`);
+            }
+        }
+        async hideKeyboardViaMidsceneIme(adb, timeoutMs) {
+            const originalIme = await adb.shell('settings get secure default_input_method');
+            const trimmedOriginalIme = originalIme.trim();
+            try {
+                await adb.shell(`ime set ${MIDSCENE_IME_ID}`);
+                await (0, core_utils_namespaceObject.sleep)(500);
+                await adb.shell(`am broadcast -a ${MIDSCENE_IME_DISMISS_ACTION}`);
+                await (0, core_utils_namespaceObject.sleep)(500);
+                if (trimmedOriginalIme && trimmedOriginalIme !== MIDSCENE_IME_ID) await adb.shell(`ime set ${trimmedOriginalIme}`);
+                if (await this.waitForKeyboardHidden(adb, timeoutMs)) {
+                    debugDevice('Keyboard hidden successfully with MidsceneIME');
+                    return true;
+                }
+                debugDevice('MidsceneIME did not dismiss keyboard');
+                return false;
+            } catch (e) {
+                if (trimmedOriginalIme && trimmedOriginalIme !== MIDSCENE_IME_ID) try {
+                    await adb.shell(`ime set ${trimmedOriginalIme}`);
+                } catch  {}
+                throw e;
+            }
+        }
         async hideKeyboard(options, timeoutMs = 1000) {
             const adb = await this.getAdb();
-            const keyboardDismissStrategy = options?.keyboardDismissStrategy ?? this.options?.keyboardDismissStrategy ?? 'esc-first';
+            const strategy = this.resolveAutoDismissKeyboard(options);
+            if (false === strategy) return false;
             const keyboardStatus = await adb.isSoftKeyboardPresent();
             const isKeyboardShown = 'boolean' == typeof keyboardStatus ? keyboardStatus : keyboardStatus?.isKeyboardShown;
             if (!isKeyboardShown) {
                 debugDevice('Keyboard has no UI; no closing necessary');
                 return false;
             }
-            const keyCodes = 'back-first' === keyboardDismissStrategy ? [
-                4,
-                111
-            ] : [
-                111,
-                4
-            ];
-            for (const keyCode of keyCodes){
-                await adb.keyevent(keyCode);
-                const startTime = Date.now();
-                const intervalMs = 100;
-                while(Date.now() - startTime < timeoutMs){
-                    await (0, core_utils_namespaceObject.sleep)(intervalMs);
-                    const currentStatus = await adb.isSoftKeyboardPresent();
-                    const isStillShown = 'boolean' == typeof currentStatus ? currentStatus : currentStatus?.isKeyboardShown;
-                    if (!isStillShown) {
-                        debugDevice(`Keyboard hidden successfully with keycode ${keyCode}`);
-                        return true;
-                    }
+            if ('midscene-ime' === strategy || 'midscene-ime-auto-install' === strategy) {
+                const autoApprove = 'midscene-ime-auto-install' === strategy;
+                try {
+                    await this.ensureMidsceneImeInstalled(adb, autoApprove);
+                    const hidden = await this.hideKeyboardViaMidsceneIme(adb, timeoutMs);
+                    if (!hidden) console.warn('Warning: MidsceneIME was invoked but the software keyboard is still visible');
+                    return hidden;
+                } catch (e) {
+                    const installHint = autoApprove ? 'Failed to use MidsceneIME auto-install for keyboard dismissal. MidsceneIME will not fall back to ESC/BACK because that can clear user input. Please enable USB installs on the device or preinstall the helper APK manually: adb install <path-to-midscene-ime.apk>.' : "Failed to use MidsceneIME for keyboard dismissal. Please install the APK manually: adb install <path-to-midscene-ime.apk>, or use autoDismissKeyboard: 'midscene-ime-auto-install' with an AndroidAgent to auto-approve installation prompts.";
+                    throw new Error(`${installHint} Original error: ${e}`);
                 }
-                debugDevice(`Keyboard still shown after keycode ${keyCode}, trying next key`);
             }
-            console.warn('Warning: Failed to hide the software keyboard after trying both ESC and BACK keys');
+            const keyCode = 'back' === strategy ? 4 : 111;
+            await adb.keyevent(keyCode);
+            if (await this.waitForKeyboardHidden(adb, timeoutMs)) {
+                debugDevice(`Keyboard hidden successfully with keycode ${keyCode} (${strategy})`);
+                return true;
+            }
+            const fallbackKeyCode = 'back' === strategy ? 111 : 4;
+            await adb.keyevent(fallbackKeyCode);
+            if (await this.waitForKeyboardHidden(adb, timeoutMs)) {
+                debugDevice(`Keyboard hidden successfully with fallback keycode ${fallbackKeyCode}`);
+                return true;
+            }
+            console.warn('Warning: Failed to hide the software keyboard after trying all strategies');
             return false;
         }
         constructor(deviceId, options){
             device_define_property(this, "deviceId", void 0);
             device_define_property(this, "yadbPushed", false);
+            device_define_property(this, "midsceneImeInstalled", false);
+            device_define_property(this, "installApprovalHandler", void 0);
             device_define_property(this, "devicePixelRatio", 1);
             device_define_property(this, "devicePixelRatioInitialized", false);
             device_define_property(this, "adb", null);
@@ -1913,6 +2022,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             super(device, opts), agent_define_property(this, "back", void 0), agent_define_property(this, "home", void 0), agent_define_property(this, "recentApps", void 0), agent_define_property(this, "appNameMapping", void 0);
             this.appNameMapping = (0, utils_namespaceObject.mergeAndNormalizeAppNameMapping)(defaultAppNameMapping, opts?.appNameMapping);
             device.setAppNameMapping(this.appNameMapping);
+            device.setInstallApprovalHandler(async ()=>{
+                await this.aiAct('A system dialog is asking to approve USB installation. Click the "Continue Install", "Allow", "Install" or similar approval button.');
+            });
             this.back = this.createActionWrapper('AndroidBackButton');
             this.home = this.createActionWrapper('AndroidHomeButton');
             this.recentApps = this.createActionWrapper('AndroidRecentAppsButton');
@@ -1990,7 +2102,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         constructor(toolsManager){
             super({
                 name: '@midscene/android-mcp',
-                version: "1.7.0",
+                version: "1.7.1-beta-20260408073050.0",
                 description: 'Control the Android device using natural language commands'
             }, toolsManager);
         }

package/dist/types/index.d.ts

@@ -75,6 +75,8 @@ export declare interface AndroidConnectedDevice extends Device {
 export declare class AndroidDevice implements AbstractInterface {
     private deviceId;
     private yadbPushed;
+    private midsceneImeInstalled;
+    private installApprovalHandler?;
     private devicePixelRatio;
     private devicePixelRatioInitialized;
     private adb;
@@ -111,6 +113,12 @@ export declare class AndroidDevice implements AbstractInterface {
      * Set the app name to package name mapping
      */
     setAppNameMapping(mapping: Record<string, string>): void;
+    /**
+     * Set a handler that will be called when APK installation needs user approval
+     * (e.g. on MIUI devices). The handler should use aiAct or similar to click
+     * the approval button on screen.
+     */
+    setInstallApprovalHandler(handler: () => Promise<void>): void;
     /**
      * Resolve app name to package name using the mapping
      * Comparison is case-insensitive and ignores spaces, dashes, and underscores.
@@ -233,6 +241,10 @@ export declare class AndroidDevice implements AbstractInterface {
     pullUp(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
     private getDisplayArg;
     getPhysicalDisplayId(): Promise<string | null>;
+    private resolveAutoDismissKeyboard;
+    private waitForKeyboardHidden;
+    private ensureMidsceneImeInstalled;
+    private hideKeyboardViaMidsceneIme;
     hideKeyboard(options?: AndroidDeviceInputOpt, timeoutMs?: number): Promise<boolean>;
 }
 

package/dist/types/mcp-server.d.ts

@@ -67,6 +67,8 @@ declare type AndroidAgentOpt = AgentOpt & {
 declare class AndroidDevice implements AbstractInterface {
     private deviceId;
     private yadbPushed;
+    private midsceneImeInstalled;
+    private installApprovalHandler?;
     private devicePixelRatio;
     private devicePixelRatioInitialized;
     private adb;
@@ -103,6 +105,12 @@ declare class AndroidDevice implements AbstractInterface {
      * Set the app name to package name mapping
      */
     setAppNameMapping(mapping: Record<string, string>): void;
+    /**
+     * Set a handler that will be called when APK installation needs user approval
+     * (e.g. on MIUI devices). The handler should use aiAct or similar to click
+     * the approval button on screen.
+     */
+    setInstallApprovalHandler(handler: () => Promise<void>): void;
     /**
      * Resolve app name to package name using the mapping
      * Comparison is case-insensitive and ignores spaces, dashes, and underscores.
@@ -225,6 +233,10 @@ declare class AndroidDevice implements AbstractInterface {
     pullUp(startPoint?: Point, distance?: number, duration?: number): Promise<void>;
     private getDisplayArg;
     getPhysicalDisplayId(): Promise<string | null>;
+    private resolveAutoDismissKeyboard;
+    private waitForKeyboardHidden;
+    private ensureMidsceneImeInstalled;
+    private hideKeyboardViaMidsceneIme;
     hideKeyboard(options?: AndroidDeviceInputOpt, timeoutMs?: number): Promise<boolean>;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/android",
-  "version": "1.7.0",
+  "version": "1.7.1-beta-20260408073050.0",
   "description": "Android automation library for Midscene",
   "keywords": [
     "Android UI automation",
@@ -41,8 +41,8 @@
     "@yume-chan/stream-extra": "2.1.0",
     "appium-adb": "12.12.1",
     "sharp": "^0.34.3",
-    "@midscene/core": "1.7.0",
-    "@midscene/shared": "1.7.0"
+    "@midscene/core": "1.7.1-beta-20260408073050.0",
+    "@midscene/shared": "1.7.1-beta-20260408073050.0"
   },
   "optionalDependencies": {
     "@ffmpeg-installer/ffmpeg": "^1.1.0"
@@ -56,12 +56,12 @@
     "tsx": "^4.19.2",
     "vitest": "3.0.5",
     "zod": "3.24.3",
-    "@midscene/playground": "1.7.0"
+    "@midscene/playground": "1.7.1-beta-20260408073050.0"
   },
   "license": "MIT",
   "scripts": {
     "dev": "npm run build:watch",
-    "prebuild": "node scripts/download-scrcpy-server.mjs && node scripts/download-yadb.mjs",
+    "prebuild": "node scripts/download-scrcpy-server.mjs && node scripts/download-yadb.mjs && node scripts/build-midscene-ime.mjs",
     "build": "rslib build",
     "build:watch": "rslib build --watch --no-clean",
     "playground": "DEBUG=midscene:* tsx demo/playground.ts",