@midscene/android 1.10.8 → 1.10.9-beta-20260731103801.0

package/dist/es/index.mjs

@@ -98,20 +98,12 @@ var __webpack_modules__ = {
                 try {
                     this.isConnecting = true;
                     debugScrcpy('Starting scrcpy connection...');
-                    const { AdbScrcpyClient, AdbScrcpyOptions3_3_3 } = await import("@yume-chan/adb-scrcpy");
+                    const { AdbScrcpyClient } = await import("@yume-chan/adb-scrcpy");
                     const { ReadableStream } = await import("@yume-chan/stream-extra");
                     const { DefaultServerPath } = await import("@yume-chan/scrcpy");
                     const serverBinPath = this.resolveServerBinPath();
                     await AdbScrcpyClient.pushServer(this.adb, ReadableStream.from((0, node_fs__rspack_import_0.createReadStream)(serverBinPath)));
-                    const scrcpyOptions = new AdbScrcpyOptions3_3_3({
-                        audio: false,
-                        control: false,
-                        maxSize: this.options.maxSize,
-                        videoBitRate: this.options.videoBitRate,
-                        maxFps: 10,
-                        sendFrameMeta: true,
-                        videoCodecOptions: 'i-frame-interval=0,bitrate-mode=2'
-                    });
+                    const scrcpyOptions = await this.createScrcpyOptions();
                     this.scrcpyClient = await AdbScrcpyClient.start(this.adb, DefaultServerPath, scrcpyOptions);
                     serverOutputTask = this.collectServerOutput(this.scrcpyClient.output, serverOutput);
                     const videoStreamPromise = this.scrcpyClient.videoStream;
@@ -139,6 +131,23 @@ var __webpack_modules__ = {
                     this.isConnecting = false;
                 }
             }
+            async createScrcpyOptions() {
+                const [{ AdbScrcpyOptions3_3_3 }, { ScrcpyInstanceId }] = await Promise.all([
+                    import("@yume-chan/adb-scrcpy"),
+                    import("@yume-chan/scrcpy")
+                ]);
+                return new AdbScrcpyOptions3_3_3({
+                    audio: false,
+                    control: false,
+                    tunnelForward: true,
+                    scid: ScrcpyInstanceId.random(),
+                    maxSize: this.options.maxSize,
+                    videoBitRate: this.options.videoBitRate,
+                    maxFps: 10,
+                    sendFrameMeta: true,
+                    videoCodecOptions: 'i-frame-interval=0,bitrate-mode=2'
+                });
+            }
             async collectServerOutput(output, lines) {
                 const reader = output.getReader();
                 try {
@@ -783,6 +792,230 @@ class ScrcpyDeviceAdapter {
         this.retryAfter = null;
     }
 }
+const TAG_RE = /<(\/?)([A-Za-z_][A-Za-z0-9_.\-:]*)([^>]*?)(\/?)>/g;
+const ATTR_RE = /([A-Za-z_][A-Za-z0-9_.\-:]*)\s*=\s*("([^"]*)"|'([^']*)')/g;
+const BOUNDS_RE = /^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$/;
+const ENTITY_TABLE = {
+    amp: '&',
+    lt: '<',
+    gt: '>',
+    quot: '"',
+    apos: "'"
+};
+function decodeEntities(value) {
+    if (!value.includes('&')) return value;
+    return value.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (full, body)=>{
+        if (body.startsWith('#x') || body.startsWith('#X')) {
+            const code = Number.parseInt(body.slice(2), 16);
+            return Number.isFinite(code) ? String.fromCodePoint(code) : full;
+        }
+        if (body.startsWith('#')) {
+            const code = Number.parseInt(body.slice(1), 10);
+            return Number.isFinite(code) ? String.fromCodePoint(code) : full;
+        }
+        return ENTITY_TABLE[body] ?? full;
+    });
+}
+function parseAttrs(raw) {
+    const attrs = {};
+    ATTR_RE.lastIndex = 0;
+    let match;
+    while(match = ATTR_RE.exec(raw))attrs[match[1]] = decodeEntities(match[3] ?? match[4] ?? '');
+    return attrs;
+}
+function parseXml(input) {
+    if (0 === input.trim().length) throw new Error('parseAndroidUITreeXml: input is empty');
+    const cleaned = input.replace(/<\?xml[\s\S]*?\?>/g, '').replace(/<!DOCTYPE[\s\S]*?>/gi, '').replace(/<!--[\s\S]*?-->/g, '');
+    TAG_RE.lastIndex = 0;
+    const stack = [];
+    let root;
+    let match;
+    while(match = TAG_RE.exec(cleaned)){
+        const isClose = '/' === match[1];
+        const name = match[2];
+        if (isClose) {
+            const current = stack.pop();
+            if (!current || current.name !== name) throw new Error(`parseAndroidUITreeXml: unbalanced close tag </${name}>`);
+            continue;
+        }
+        const element = {
+            name,
+            attrs: parseAttrs(match[3] ?? ''),
+            children: []
+        };
+        const parent = stack[stack.length - 1];
+        if (parent) parent.children.push(element);
+        else if (root) throw new Error('parseAndroidUITreeXml: more than one top-level element');
+        else root = element;
+        if ('/' !== match[4]) stack.push(element);
+    }
+    if (stack.length > 0) throw new Error('parseAndroidUITreeXml: unclosed element');
+    if (!root) throw new Error('parseAndroidUITreeXml: no root element');
+    return root;
+}
+function parseBounds(raw, devicePixelRatio) {
+    const match = raw ? BOUNDS_RE.exec(raw) : void 0;
+    if (!match) return {
+        left: 0,
+        top: 0,
+        width: 0,
+        height: 0
+    };
+    const left = Number.parseInt(match[1], 10);
+    const top = Number.parseInt(match[2], 10);
+    const right = Number.parseInt(match[3], 10);
+    const bottom = Number.parseInt(match[4], 10);
+    const ratio = devicePixelRatio > 0 ? devicePixelRatio : 1;
+    return {
+        left: left / ratio,
+        top: top / ratio,
+        width: Math.max(0, right - left) / ratio,
+        height: Math.max(0, bottom - top) / ratio
+    };
+}
+function toUiNode(element, devicePixelRatio) {
+    const attrs = {};
+    for (const [name, value] of Object.entries(element.attrs))if ('bounds' !== name) attrs[name] = value;
+    return {
+        type: element.attrs.class || element.name,
+        attrs,
+        bounds: parseBounds(element.attrs.bounds, devicePixelRatio),
+        children: element.children.map((child)=>toUiNode(child, devicePixelRatio))
+    };
+}
+function uiautomatorXmlToUiNode(xml, devicePixelRatio) {
+    const root = parseXml(xml);
+    if ('hierarchy' === root.name && 1 === root.children.length) return toUiNode(root.children[0], devicePixelRatio);
+    return toUiNode(root, devicePixelRatio);
+}
+const ACCESSIBILITY_DUMP_ATTEMPTS = 3;
+const ACCESSIBILITY_DUMP_TIMEOUT_MS = 5000;
+const ACCESSIBILITY_DUMP_RETRY_DELAY_MS = 250;
+const UIAUTOMATOR_LAYOUT_PATH = '/sdcard/midscene_window_dump.xml';
+const debugUITree = (0, logger_.getDebug)('android:ui-tree');
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function validateHierarchyXml(xml) {
+    if ('string' != typeof xml || !/<hierarchy(?:\s|>)/.test(xml) || !xml.includes('</hierarchy>')) throw new Error('uiautomator did not produce valid hierarchy XML');
+}
+function focusedPackageForLogicalDisplay(windowDump, logicalDisplayId) {
+    const displayMatches = [
+        ...windowDump.matchAll(/^\s*Display: mDisplayId=(\d+)\b.*$/gm)
+    ];
+    const displayIndex = displayMatches.findIndex((match)=>Number(match[1]) === logicalDisplayId);
+    if (-1 === displayIndex) return null;
+    const start = displayMatches[displayIndex].index ?? 0;
+    const end = displayMatches[displayIndex + 1]?.index ?? windowDump.length;
+    const displayBlock = windowDump.slice(start, end);
+    for (const field of [
+        'mCurrentFocus',
+        'mFocusedApp'
+    ]){
+        const value = displayBlock.match(new RegExp(`^\\s*${field}=([^\\n]*)$`, 'm'))?.[1];
+        const packageName = value?.match(/\bu\d+\s+([A-Za-z0-9_$.-]+)\//)?.[1];
+        if (packageName) return packageName;
+    }
+    return null;
+}
+function firstTreePackage(root) {
+    if (root.attrs.package) return root.attrs.package;
+    for (const child of root.children){
+        const packageName = firstTreePackage(child);
+        if (packageName) return packageName;
+    }
+    return null;
+}
+function windowDisplayIdsForPackage(windowDump, packageName) {
+    const windowMatches = [
+        ...windowDump.matchAll(/^\s*Window #\d+ Window\{.*$/gm)
+    ];
+    const displayIds = new Set();
+    for(let index = 0; index < windowMatches.length; index++){
+        const start = windowMatches[index].index ?? 0;
+        const end = windowMatches[index + 1]?.index ?? windowDump.length;
+        const windowBlock = windowDump.slice(start, end);
+        const ownerPackage = windowBlock.match(/^\s*mOwnerUid=.*\bpackage=([A-Za-z0-9_$.-]+)\b/m)?.[1];
+        const headerPackage = windowMatches[index][0].match(/\bu\d+\s+([A-Za-z0-9_$.-]+)\//)?.[1];
+        if (ownerPackage !== packageName && headerPackage !== packageName) continue;
+        const displayId = windowBlock.match(/^\s*mDisplayId=(\d+)\b/m)?.[1];
+        if (void 0 !== displayId) displayIds.add(Number(displayId));
+    }
+    return [
+        ...displayIds
+    ];
+}
+function uiTreeExtent(root) {
+    let right = root.bounds.left + root.bounds.width;
+    let bottom = root.bounds.top + root.bounds.height;
+    for (const child of root.children){
+        const childExtent = uiTreeExtent(child);
+        right = Math.max(right, childExtent.right);
+        bottom = Math.max(bottom, childExtent.bottom);
+    }
+    return {
+        right,
+        bottom
+    };
+}
+async function dumpAccessibilityXml(adb) {
+    await runAdbShellStdoutOrThrow(adb, `rm -f ${UIAUTOMATOR_LAYOUT_PATH}`, {
+        timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+    });
+    await runAdbShellStdoutOrThrow(adb, `uiautomator dump --compressed ${UIAUTOMATOR_LAYOUT_PATH}`, {
+        timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+    });
+    const xml = await runAdbShellStdoutOrThrow(adb, `cat ${UIAUTOMATOR_LAYOUT_PATH}`, {
+        timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+    });
+    validateHierarchyXml(xml);
+    return xml;
+}
+async function validateUITreeDisplay(root, options) {
+    if ('number' != typeof options.displayId) return;
+    const { adb, displayId } = options;
+    const displayWindowDump = await runAdbShellStdoutOrThrow(adb, 'dumpsys window displays', {
+        timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+    });
+    const expectedPackage = focusedPackageForLogicalDisplay(displayWindowDump, displayId);
+    const treePackage = firstTreePackage(root);
+    if (!expectedPackage) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: no focused package found on the target display`);
+    if (!treePackage) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: accessibility hierarchy has no package`);
+    const allWindowDump = await runAdbShellStdoutOrThrow(adb, 'dumpsys window windows', {
+        timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
+    });
+    const treePackageDisplayIds = windowDisplayIdsForPackage(allWindowDump, treePackage);
+    if (0 === treePackageDisplayIds.length) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: no window display ownership found for accessibility package ${treePackage}`);
+    if (!treePackageDisplayIds.includes(displayId)) throw new Error(`Android UI tree display mismatch for displayId=${displayId}: expected focused package ${expectedPackage}, but accessibility hierarchy belongs to ${treePackage} on displayId=${treePackageDisplayIds.join(',')}`);
+    const targetSize = await options.getDisplaySize();
+    const extent = uiTreeExtent(root);
+    const tolerance = 1;
+    if (extent.right > targetSize.width + tolerance || extent.bottom > targetSize.height + tolerance) throw new Error(`Android UI tree bounds exceed displayId=${displayId}: tree ${Math.round(extent.right)}x${Math.round(extent.bottom)} is outside target ${targetSize.width}x${targetSize.height}`);
+}
+async function captureAndroidUITree(options) {
+    const failures = [];
+    for(let attempt = 1; attempt <= ACCESSIBILITY_DUMP_ATTEMPTS; attempt++){
+        const startedAt = Date.now();
+        try {
+            const xml = await dumpAccessibilityXml(options.adb);
+            const capturedAt = Date.now();
+            const root = uiautomatorXmlToUiNode(xml, options.devicePixelRatio);
+            await validateUITreeDisplay(root, options);
+            debugUITree('capture source=%s attempt=%d durationMs=%d bytes=%d', 'uiautomator', attempt, Date.now() - startedAt, xml.length);
+            return {
+                platform: 'android',
+                capturedAt,
+                root
+            };
+        } catch (error) {
+            const failure = `uiautomator attempt ${attempt}/${ACCESSIBILITY_DUMP_ATTEMPTS}: ${errorMessage(error)}`;
+            failures.push(failure);
+            debugUITree('capture failed source=%s attempt=%d/%d durationMs=%d error=%s', 'uiautomator', attempt, ACCESSIBILITY_DUMP_ATTEMPTS, Date.now() - startedAt, errorMessage(error));
+            if (attempt < ACCESSIBILITY_DUMP_ATTEMPTS) await sleep(ACCESSIBILITY_DUMP_RETRY_DELAY_MS);
+        }
+    }
+    throw new Error(`Unable to capture Android UI tree after ${failures.length} attempt(s): ${failures.join('; ')}`);
+}
 function device_define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -802,6 +1035,35 @@ const debugDevice = (0, logger_.getDebug)('android:device');
 const warnDevice = (0, logger_.getDebug)('android:device', {
     console: true
 });
+function displayViewportForLogicalDisplay(displayDump, logicalDisplayId) {
+    for (const viewportMatch of displayDump.matchAll(/DisplayViewport\{([^}]*)\}/g)){
+        const viewport = viewportMatch[1];
+        const displayId = viewport.match(/\bdisplayId=(\d+)\b/)?.[1];
+        if (Number(displayId) === logicalDisplayId) return viewportMatch[0];
+    }
+    return null;
+}
+function displayInfoLineForPhysicalDisplay(displayDump, physicalDisplayId) {
+    const lineRegex = new RegExp(`^.*uniqueId ["']local:${physicalDisplayId}["'].*$`, 'm');
+    return displayDump.match(lineRegex)?.[0] ?? null;
+}
+function physicalDisplayIdForLogicalDisplay(displayDump, logicalDisplayId) {
+    const viewport = displayViewportForLogicalDisplay(displayDump, logicalDisplayId);
+    if (viewport) {
+        const physicalId = viewport.match(/\buniqueId=['"]local:(\d+)['"]/)?.[1];
+        if (physicalId) return physicalId;
+    }
+    const lines = displayDump.split(/\r?\n/);
+    for(let index = 0; index < lines.length; index++){
+        const displayId = lines[index].match(/^\s*mDisplayId=(\d+)\b/)?.[1];
+        if (Number(displayId) === logicalDisplayId) for(let cursor = index + 1; cursor < lines.length; cursor++){
+            if (/^\s*mDisplayId=\d+\b/.test(lines[cursor])) break;
+            const physicalId = lines[cursor].match(/mBaseDisplayInfo=.*\buniqueId "local:(\d+)"/)?.[1];
+            if (physicalId) return physicalId;
+        }
+    }
+    return null;
+}
 function escapeForShell(text) {
     return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
 }
@@ -1089,6 +1351,15 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             children: []
         };
     }
+    async getUITree() {
+        await this.initializeDevicePixelRatio();
+        return captureAndroidUITree({
+            adb: await this.getAdb(),
+            devicePixelRatio: this.devicePixelRatio,
+            displayId: this.options?.displayId,
+            getDisplaySize: ()=>this.size()
+        });
+    }
     async getScreenSize() {
         const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
         debugDevice(`getScreenSize: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedSize=${!!this.cachedScreenSize}`);
@@ -1097,13 +1368,10 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         if ('number' == typeof this.options?.displayId) try {
             const stdout = await adb.shell('dumpsys display');
             if (this.options?.usePhysicalDisplayIdForDisplayLookup) {
-                const physicalDisplayId = await this.getPhysicalDisplayId();
+                const physicalDisplayId = await this.resolvePhysicalDisplayId(stdout);
                 if (physicalDisplayId) {
-                    const lineRegex = new RegExp(`^.*uniqueId \"local:${physicalDisplayId}\".*$
-`, 'm');
-                    const lineMatch = stdout.match(lineRegex);
-                    if (lineMatch) {
-                        const targetLine = lineMatch[0];
+                    const targetLine = displayInfoLineForPhysicalDisplay(stdout, physicalDisplayId);
+                    if (targetLine) {
                         const realMatch = targetLine.match(/real (\d+) x (\d+)/);
                         const rotationMatch = targetLine.match(/rotation (\d+)/);
                         if (realMatch && rotationMatch) {
@@ -1124,10 +1392,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
                     }
                 }
             } else {
-                const viewportRegex = new RegExp(`DisplayViewport{[^}]*displayId=${this.options.displayId}[^}]*}`, 'g');
-                const match = stdout.match(viewportRegex);
-                if (match) {
-                    const targetLine = match[0];
+                const targetLine = displayViewportForLogicalDisplay(stdout, this.options.displayId);
+                if (targetLine) {
                     const physicalFrameMatch = targetLine.match(/physicalFrame=Rect\(\d+, \d+ - (\d+), (\d+)\)/);
                     const orientationMatch = targetLine.match(/orientation=(\d+)/);
                     if (physicalFrameMatch && orientationMatch) {
@@ -1193,13 +1459,10 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         if ('number' == typeof this.options?.displayId) try {
             const stdout = await adb.shell('dumpsys display');
             if (this.options?.usePhysicalDisplayIdForDisplayLookup) {
-                const physicalDisplayId = await this.getPhysicalDisplayId();
+                const physicalDisplayId = await this.resolvePhysicalDisplayId(stdout);
                 if (physicalDisplayId) {
-                    const lineRegex = new RegExp(`^.*uniqueId \"local:${physicalDisplayId}\".*$
-`, 'm');
-                    const lineMatch = stdout.match(lineRegex);
-                    if (lineMatch) {
-                        const targetLine = lineMatch[0];
+                    const targetLine = displayInfoLineForPhysicalDisplay(stdout, physicalDisplayId);
+                    if (targetLine) {
                         const densityMatch = targetLine.match(/density (\d+)/);
                         if (densityMatch) {
                             const density = Number(densityMatch[1]);
@@ -1337,7 +1600,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
             debugDevice('screenshotBase64 end (scrcpy mode)');
             return result;
         } catch (error) {
-            debugDevice(`Scrcpy screenshot failed, falling back to standard ADB method.\nError: ${error}`);
+            warnDevice(`Scrcpy screenshot failed, falling back to standard ADB method.\nError: ${error}`);
         }
         const adb = await this.getAdb();
         let screenshotBuffer;
@@ -1824,6 +2087,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         if ('number' == typeof this.options?.displayId && 0 !== this.options.displayId) warnDevice(`yadb ${operation} cannot target display ${this.options.displayId}; it will act on the default display.`);
     }
     async getPhysicalDisplayId() {
+        return this.resolvePhysicalDisplayId();
+    }
+    async resolvePhysicalDisplayId(displayDump) {
         if (void 0 !== this.cachedPhysicalDisplayId) return this.cachedPhysicalDisplayId;
         if ('number' != typeof this.options?.displayId) {
             this.cachedPhysicalDisplayId = null;
@@ -1831,6 +2097,13 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>`  ${key} size: ${size[k
         }
         const adb = await this.getAdb();
         try {
+            const resolvedDisplayDump = displayDump ?? await adb.shell('dumpsys display');
+            const logicalDisplayPhysicalId = physicalDisplayIdForLogicalDisplay(resolvedDisplayDump, this.options.displayId);
+            if (logicalDisplayPhysicalId) {
+                this.cachedPhysicalDisplayId = logicalDisplayPhysicalId;
+                debugDevice(`Found and cached physical display ID: ${logicalDisplayPhysicalId} for logical display ID: ${this.options.displayId}`);
+                return this.cachedPhysicalDisplayId;
+            }
             const stdout = await adb.shell(`dumpsys SurfaceFlinger --display-id ${this.options.displayId}`);
             const regex = new RegExp(`Display (\\d+) \\(HWC display ${this.options.displayId}\\):`);
             const displayMatch = stdout.match(regex);