@midscene/android 1.10.8 → 1.10.9-beta-20260731103801.0

package/dist/lib/cli.js

@@ -87,20 +87,12 @@ var __webpack_modules__ = {
                 try {
                     this.isConnecting = true;
                     debugScrcpy('Starting scrcpy connection...');
-                    const { AdbScrcpyClient, AdbScrcpyOptions3_3_3 } = await import("@yume-chan/adb-scrcpy");
+                    const { AdbScrcpyClient } = await import("@yume-chan/adb-scrcpy");
                     const { ReadableStream } = await import("@yume-chan/stream-extra");
                     const { DefaultServerPath } = await import("@yume-chan/scrcpy");
                     const serverBinPath = this.resolveServerBinPath();
                     await AdbScrcpyClient.pushServer(this.adb, ReadableStream.from((0, node_fs__rspack_import_0.createReadStream)(serverBinPath)));
-                    const scrcpyOptions = new AdbScrcpyOptions3_3_3({
-                        audio: false,
-                        control: false,
-                        maxSize: this.options.maxSize,
-                        videoBitRate: this.options.videoBitRate,
-                        maxFps: 10,
-                        sendFrameMeta: true,
-                        videoCodecOptions: 'i-frame-interval=0,bitrate-mode=2'
-                    });
+                    const scrcpyOptions = await this.createScrcpyOptions();
                     this.scrcpyClient = await AdbScrcpyClient.start(this.adb, DefaultServerPath, scrcpyOptions);
                     serverOutputTask = this.collectServerOutput(this.scrcpyClient.output, serverOutput);
                     const videoStreamPromise = this.scrcpyClient.videoStream;
@@ -128,6 +120,23 @@ var __webpack_modules__ = {
                     this.isConnecting = false;
                 }
             }
+            async createScrcpyOptions() {
+                const [{ AdbScrcpyOptions3_3_3 }, { ScrcpyInstanceId }] = await Promise.all([
+                    import("@yume-chan/adb-scrcpy"),
+                    import("@yume-chan/scrcpy")
+                ]);
+                return new AdbScrcpyOptions3_3_3({
+                    audio: false,
+                    control: false,
+                    tunnelForward: true,
+                    scid: ScrcpyInstanceId.random(),
+                    maxSize: this.options.maxSize,
+                    videoBitRate: this.options.videoBitRate,
+                    maxFps: 10,
+                    sendFrameMeta: true,
+                    videoCodecOptions: 'i-frame-interval=0,bitrate-mode=2'
+                });
+            }
             async collectServerOutput(output, lines) {
                 const reader = output.getReader();
                 try {
@@ -895,6 +904,230 @@ ${stdout ? truncateAdbShellStream(stdout, 'stdout') : EMPTY_ADB_SHELL_STDOUT}`;
             this.retryAfter = null;
         }
     }
+    const TAG_RE = /<(\/?)([A-Za-z_][A-Za-z0-9_.\-:]*)([^>]*?)(\/?)>/g;
+    const ATTR_RE = /([A-Za-z_][A-Za-z0-9_.\-:]*)\s*=\s*("([^"]*)"|'([^']*)')/g;
+    const BOUNDS_RE = /^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$/;
+    const ENTITY_TABLE = {
+        amp: '&',
+        lt: '<',
+        gt: '>',
+        quot: '"',
+        apos: "'"
+    };
+    function decodeEntities(value) {
+        if (!value.includes('&')) return value;
+        return value.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (full, body)=>{
+            if (body.startsWith('#x') || body.startsWith('#X')) {
+                const code = Number.parseInt(body.slice(2), 16);
+                return Number.isFinite(code) ? String.fromCodePoint(code) : full;
+            }
+            if (body.startsWith('#')) {
+                const code = Number.parseInt(body.slice(1), 10);
+                return Number.isFinite(code) ? String.fromCodePoint(code) : full;
+            }
+            return ENTITY_TABLE[body] ?? full;
+        });
+    }
+    function parseAttrs(raw) {
+        const attrs = {};
+        ATTR_RE.lastIndex = 0;
+        let match;
+        while(match = ATTR_RE.exec(raw))attrs[match[1]] = decodeEntities(match[3] ?? match[4] ?? '');
+        return attrs;
+    }
+    function parseXml(input) {
+        if (0 === input.trim().length) throw new Error('parseAndroidUITreeXml: input is empty');
+        const cleaned = input.replace(/<\?xml[\s\S]*?\?>/g, '').replace(/<!DOCTYPE[\s\S]*?>/gi, '').replace(/<!--[\s\S]*?-->/g, '');
+        TAG_RE.lastIndex = 0;
+        const stack = [];
+        let root;
+        let match;
+        while(match = TAG_RE.exec(cleaned)){
+            const isClose = '/' === match[1];
+            const name = match[2];
+            if (isClose) {
+                const current = stack.pop();
+                if (!current || current.name !== name) throw new Error(`parseAndroidUITreeXml: unbalanced close tag </${name}>`);
+                continue;
+            }
+            const element = {
+                name,
+                attrs: parseAttrs(match[3] ?? ''),
+                children: []
+            };
+            const parent = stack[stack.length - 1];
+            if (parent) parent.children.push(element);
+            else if (root) throw new Error('parseAndroidUITreeXml: more than one top-level element');
+            else root = element;
+            if ('/' !== match[4]) stack.push(element);
+        }
+        if (stack.length > 0) throw new Error('parseAndroidUITreeXml: unclosed element');
+        if (!root) throw new Error('parseAndroidUITreeXml: no root element');
+        return root;
+    }
+    function parseBounds(raw, devicePixelRatio) {
+        const match = raw ? BOUNDS_RE.exec(raw) : void 0;
+        if (!match) return {
+            left: 0,
+            top: 0,
+            width: 0,
+            height: 0
+        };
+        const left = Number.parseInt(match[1], 10);
+        const top = Number.parseInt(match[2], 10);
+        const right = Number.parseInt(match[3], 10);
+        const bottom = Number.parseInt(match[4], 10);
+        const ratio = devicePixelRatio > 0 ? devicePixelRatio : 1;
+        return {
+            left: left / ratio,
+            top: top / ratio,
+            width: Math.max(0, right - left) / ratio,
+            height: Math.max(0, bottom - top) / ratio
+        };
+    }
+    function toUiNode(element, devicePixelRatio) {
+        const attrs = {};
+        for (const [name, value] of Object.entries(element.attrs))if ('bounds' !== name) attrs[name] = value;
+        return {
+            type: element.attrs.class || element.name,
+            attrs,
+            bounds: parseBounds(element.attrs.bounds, devicePixelRatio),
+            children: element.children.map((child)=>toUiNode(child, devicePixelRatio))
+        };
+    }
+    function uiautomatorXmlToUiNode(xml, devicePixelRatio) {
+        const root = parseXml(xml);
+        if ('hierarchy' === root.name && 1 === root.children.length) return toUiNode(root.children[0], devicePixelRatio);
+        return toUiNode(root, devicePixelRatio);
+    }
+    const ACCESSIBILITY_DUMP_ATTEMPTS = 3;
+    const ACCESSIBILITY_DUMP_TIMEOUT_MS = 5000;
+    const ACCESSIBILITY_DUMP_RETRY_DELAY_MS = 250;
+    const UIAUTOMATOR_LAYOUT_PATH = '/sdcard/midscene_window_dump.xml';
+    const debugUITree = (0, logger_.getDebug)('android:ui-tree');
+    function errorMessage(error) {
+        return error instanceof Error ? error.message : String(error);
+    }
+    function validateHierarchyXml(xml) {
+        if ('string' != typeof xml || !/<hierarchy(?:\s|>)/.test(xml) || !xml.includes('</hierarchy>')) throw new Error('uiautomator did not produce valid hierarchy XML');
+    }
+    function focusedPackageForLogicalDisplay(windowDump, logicalDisplayId) {
+        const displayMatches = [
+            ...windowDump.matchAll(/^\s*Display: mDisplayId=(\d+)\b.*$/gm)
+        ];
+        const displayIndex = displayMatches.findIndex((match)=>Number(match[1]) === logicalDisplayId);
+        if (-1 === displayIndex) return null;
+        const start = displayMatches[displayIndex].index ?? 0;
+        const end = displayMatches[displayIndex + 1]?.index ?? windowDump.length;
+        const displayBlock = windowDump.slice(start, end);
+        for (const field of [
+            'mCurrentFocus',
+            'mFocusedApp'
+        ]){
+            const value = displayBlock.match(new RegExp(`^\\s*${field}=([^\\n]*)$`, 'm'))?.[1];
+            const packageName = value?.match(/\bu\d+\s+([A-Za-z0-9_$.-]+)\//)?.[1];
+            if (packageName) return packageName;
+        }
+        return null;
+    }
+    function firstTreePackage(root) {
+        if (root.attrs.package) return root.attrs.package;
+        for (const child of root.children){
+            const packageName = firstTreePackage(child);
+            if (packageName) return packageName;
+        }
+        return null;
+    }
+    function windowDisplayIdsForPackage(windowDump, packageName) {
+        const windowMatches = [
+            ...windowDump.matchAll(/^\s*Window #\d+ Window\{.*$/gm)
+        ];
+        const displayIds = new Set();
+        for(let index = 0; index < windowMatches.length; index++){
+            const start = windowMatches[index].index ?? 0;
+            const end = windowMatches[index + 1]?.index ?? windowDump.length;
+            const windowBlock = windowDump.slice(start, end);
+            const ownerPackage = windowBlock.match(/^\s*mOwnerUid=.*\bpackage=([A-Za-z0-9_$.-]+)\b/m)?.[1];
+            const headerPackage = windowMatches[index][0].match(/\bu\d+\s+([A-Za-z0-9_$.-]+)\//)?.[1];
+            if (ownerPackage !== packageName && headerPackage !== packageName) continue;
+            const displayId = windowBlock.match(/^\s*mDisplayId=(\d+)\b/m)?.[1];
+            if (void 0 !== displayId) displayIds.add(Number(displayId));
+        }
+        return [
+            ...displayIds
+        ];
+    }
+    function uiTreeExtent(root) {
+        let right = root.bounds.left + root.bounds.width;
+        let bottom = root.bounds.top + root.bounds.height;
+        for (const child of root.children){
+            const childExtent = uiTreeExtent(child);
+            right = Math.max(right, childExtent.right);
+            bottom = Math.max(bottom, childExtent.bottom);
+        }
+        return {
+            right,
+            bottom
+        };
+    }
+    async function dumpAccessibilityXml(adb) {
+        await runAdbShellStdoutOrThrow(adb, `rm -f ${UIAUTOMATOR_LAYOUT_PATH}`, {
+            timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+        });
+        await runAdbShellStdoutOrThrow(adb, `uiautomator dump --compressed ${UIAUTOMATOR_LAYOUT_PATH}`, {
+            timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+        });
+        const xml = await runAdbShellStdoutOrThrow(adb, `cat ${UIAUTOMATOR_LAYOUT_PATH}`, {
+            timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+        });
+        validateHierarchyXml(xml);
+        return xml;
+    }
+    async function validateUITreeDisplay(root, options) {
+        if ('number' != typeof options.displayId) return;
+        const { adb, displayId } = options;
+        const displayWindowDump = await runAdbShellStdoutOrThrow(adb, 'dumpsys window displays', {
+            timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+        });
+        const expectedPackage = focusedPackageForLogicalDisplay(displayWindowDump, displayId);
+        const treePackage = firstTreePackage(root);
+        if (!expectedPackage) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: no focused package found on the target display`);
+        if (!treePackage) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: accessibility hierarchy has no package`);
+        const allWindowDump = await runAdbShellStdoutOrThrow(adb, 'dumpsys window windows', {
+            timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+        });
+        const treePackageDisplayIds = windowDisplayIdsForPackage(allWindowDump, treePackage);
+        if (0 === treePackageDisplayIds.length) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: no window display ownership found for accessibility package ${treePackage}`);
+        if (!treePackageDisplayIds.includes(displayId)) throw new Error(`Android UI tree display mismatch for displayId=${displayId}: expected focused package ${expectedPackage}, but accessibility hierarchy belongs to ${treePackage} on displayId=${treePackageDisplayIds.join(',')}`);
+        const targetSize = await options.getDisplaySize();
+        const extent = uiTreeExtent(root);
+        const tolerance = 1;
+        if (extent.right > targetSize.width + tolerance || extent.bottom > targetSize.height + tolerance) throw new Error(`Android UI tree bounds exceed displayId=${displayId}: tree ${Math.round(extent.right)}x${Math.round(extent.bottom)} is outside target ${targetSize.width}x${targetSize.height}`);
+    }
+    async function captureAndroidUITree(options) {
+        const failures = [];
+        for(let attempt = 1; attempt <= ACCESSIBILITY_DUMP_ATTEMPTS; attempt++){
+            const startedAt = Date.now();
+            try {
+                const xml = await dumpAccessibilityXml(options.adb);
+                const capturedAt = Date.now();
+                const root = uiautomatorXmlToUiNode(xml, options.devicePixelRatio);
+                await validateUITreeDisplay(root, options);
+                debugUITree('capture source=%s attempt=%d durationMs=%d bytes=%d', 'uiautomator', attempt, Date.now() - startedAt, xml.length);
+                return {
+                    platform: 'android',
+                    capturedAt,
+                    root
+                };
+            } catch (error) {
+                const failure = `uiautomator attempt ${attempt}/${ACCESSIBILITY_DUMP_ATTEMPTS}: ${errorMessage(error)}`;
+                failures.push(failure);
+                debugUITree('capture failed source=%s attempt=%d/%d durationMs=%d error=%s', 'uiautomator', attempt, ACCESSIBILITY_DUMP_ATTEMPTS, Date.now() - startedAt, errorMessage(error));
+                if (attempt < ACCESSIBILITY_DUMP_ATTEMPTS) await (0, core_utils_namespaceObject.sleep)(ACCESSIBILITY_DUMP_RETRY_DELAY_MS);
+            }
+        }
+        throw new Error(`Unable to capture Android UI tree after ${failures.length} attempt(s): ${failures.join('; ')}`);
+    }
     function device_define_property(obj, key, value) {
         if (key in obj) Object.defineProperty(obj, key, {
             value: value,
@@ -914,6 +1147,35 @@ ${stdout ? truncateAdbShellStream(stdout, 'stdout') : EMPTY_ADB_SHELL_STDOUT}`;
     const warnDevice = (0, logger_.getDebug)('android:device', {
         console: true
     });
+    function displayViewportForLogicalDisplay(displayDump, logicalDisplayId) {
+        for (const viewportMatch of displayDump.matchAll(/DisplayViewport\{([^}]*)\}/g)){
+            const viewport = viewportMatch[1];
+            const displayId = viewport.match(/\bdisplayId=(\d+)\b/)?.[1];
+            if (Number(displayId) === logicalDisplayId) return viewportMatch[0];
+        }
+        return null;
+    }
+    function displayInfoLineForPhysicalDisplay(displayDump, physicalDisplayId) {
+        const lineRegex = new RegExp(`^.*uniqueId ["']local:${physicalDisplayId}["'].*$`, 'm');
+        return displayDump.match(lineRegex)?.[0] ?? null;
+    }
+    function physicalDisplayIdForLogicalDisplay(displayDump, logicalDisplayId) {
+        const viewport = displayViewportForLogicalDisplay(displayDump, logicalDisplayId);
+        if (viewport) {
+            const physicalId = viewport.match(/\buniqueId=['"]local:(\d+)['"]/)?.[1];
+            if (physicalId) return physicalId;
+        }
+        const lines = displayDump.split(/\r?\n/);
+        for(let index = 0; index < lines.length; index++){
+            const displayId = lines[index].match(/^\s*mDisplayId=(\d+)\b/)?.[1];
+            if (Number(displayId) === logicalDisplayId) for(let cursor = index + 1; cursor < lines.length; cursor++){
+                if (/^\s*mDisplayId=\d+\b/.test(lines[cursor])) break;
+                const physicalId = lines[cursor].match(/mBaseDisplayInfo=.*\buniqueId "local:(\d+)"/)?.[1];
+                if (physicalId) return physicalId;
+            }
+        }
+        return null;
+    }
     function escapeForShell(text) {
         return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
     }
@@ -1201,6 +1463,15 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                 children: []
             };
         }
+        async getUITree() {
+            await this.initializeDevicePixelRatio();
+            return captureAndroidUITree({
+                adb: await this.getAdb(),
+                devicePixelRatio: this.devicePixelRatio,
+                displayId: this.options?.displayId,
+                getDisplaySize: ()=>this.size()
+            });
+        }
         async getScreenSize() {
             const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
             debugDevice(`getScreenSize: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedSize=${!!this.cachedScreenSize}`);
@@ -1209,13 +1480,10 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             if ('number' == typeof this.options?.displayId) try {
                 const stdout = await adb.shell('dumpsys display');
                 if (this.options?.usePhysicalDisplayIdForDisplayLookup) {
-                    const physicalDisplayId = await this.getPhysicalDisplayId();
+                    const physicalDisplayId = await this.resolvePhysicalDisplayId(stdout);
                     if (physicalDisplayId) {
-                        const lineRegex = new RegExp(`^.*uniqueId \"local:${physicalDisplayId}\".*$
-`, 'm');
-                        const lineMatch = stdout.match(lineRegex);
-                        if (lineMatch) {
-                            const targetLine = lineMatch[0];
+                        const targetLine = displayInfoLineForPhysicalDisplay(stdout, physicalDisplayId);
+                        if (targetLine) {
                             const realMatch = targetLine.match(/real (\d+) x (\d+)/);
                             const rotationMatch = targetLine.match(/rotation (\d+)/);
                             if (realMatch && rotationMatch) {
@@ -1236,10 +1504,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                         }
                     }
                 } else {
-                    const viewportRegex = new RegExp(`DisplayViewport{[^}]*displayId=${this.options.displayId}[^}]*}`, 'g');
-                    const match = stdout.match(viewportRegex);
-                    if (match) {
-                        const targetLine = match[0];
+                    const targetLine = displayViewportForLogicalDisplay(stdout, this.options.displayId);
+                    if (targetLine) {
                         const physicalFrameMatch = targetLine.match(/physicalFrame=Rect\(\d+, \d+ - (\d+), (\d+)\)/);
                         const orientationMatch = targetLine.match(/orientation=(\d+)/);
                         if (physicalFrameMatch && orientationMatch) {
@@ -1305,13 +1571,10 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             if ('number' == typeof this.options?.displayId) try {
                 const stdout = await adb.shell('dumpsys display');
                 if (this.options?.usePhysicalDisplayIdForDisplayLookup) {
-                    const physicalDisplayId = await this.getPhysicalDisplayId();
+                    const physicalDisplayId = await this.resolvePhysicalDisplayId(stdout);
                     if (physicalDisplayId) {
-                        const lineRegex = new RegExp(`^.*uniqueId \"local:${physicalDisplayId}\".*$
-`, 'm');
-                        const lineMatch = stdout.match(lineRegex);
-                        if (lineMatch) {
-                            const targetLine = lineMatch[0];
+                        const targetLine = displayInfoLineForPhysicalDisplay(stdout, physicalDisplayId);
+                        if (targetLine) {
                             const densityMatch = targetLine.match(/density (\d+)/);
                             if (densityMatch) {
                                 const density = Number(densityMatch[1]);
@@ -1449,7 +1712,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                 debugDevice('screenshotBase64 end (scrcpy mode)');
                 return result;
             } catch (error) {
-                debugDevice(`Scrcpy screenshot failed, falling back to standard ADB method.\nError: ${error}`);
+                warnDevice(`Scrcpy screenshot failed, falling back to standard ADB method.\nError: ${error}`);
             }
             const adb = await this.getAdb();
             let screenshotBuffer;
@@ -1936,6 +2199,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             if ('number' == typeof this.options?.displayId && 0 !== this.options.displayId) warnDevice(`yadb ${operation} cannot target display ${this.options.displayId}; it will act on the default display.`);
         }
         async getPhysicalDisplayId() {
+            return this.resolvePhysicalDisplayId();
+        }
+        async resolvePhysicalDisplayId(displayDump) {
             if (void 0 !== this.cachedPhysicalDisplayId) return this.cachedPhysicalDisplayId;
             if ('number' != typeof this.options?.displayId) {
                 this.cachedPhysicalDisplayId = null;
@@ -1943,6 +2209,13 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             }
             const adb = await this.getAdb();
             try {
+                const resolvedDisplayDump = displayDump ?? await adb.shell('dumpsys display');
+                const logicalDisplayPhysicalId = physicalDisplayIdForLogicalDisplay(resolvedDisplayDump, this.options.displayId);
+                if (logicalDisplayPhysicalId) {
+                    this.cachedPhysicalDisplayId = logicalDisplayPhysicalId;
+                    debugDevice(`Found and cached physical display ID: ${logicalDisplayPhysicalId} for logical display ID: ${this.options.displayId}`);
+                    return this.cachedPhysicalDisplayId;
+                }
                 const stdout = await adb.shell(`dumpsys SurfaceFlinger --display-id ${this.options.displayId}`);
                 const regex = new RegExp(`Display (\\d+) \\(HWC display ${this.options.displayId}\\):`);
                 const displayMatch = stdout.match(regex);
@@ -2327,7 +2600,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
     const tools = new AndroidMidsceneTools();
     (0, cli_namespaceObject.runToolsCLI)(tools, 'midscene-android', {
         stripPrefix: 'android_',
-        version: "1.10.8",
+        version: "1.10.9-beta-20260731103801.0",
         extraCommands: (0, core_namespaceObject.createReportCliCommands)()
     }).catch((e)=>{
         process.exit((0, cli_namespaceObject.reportCLIError)(e));