@midscene/android 1.10.8 → 1.10.9-beta-20260731103801.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/cli.mjs +301 -28
- package/dist/es/index.mjs +300 -27
- package/dist/lib/cli.js +301 -28
- package/dist/lib/index.js +300 -27
- package/dist/types/index.d.ts +5 -2
- package/package.json +4 -4
package/dist/lib/index.js
CHANGED
|
@@ -87,20 +87,12 @@ var __webpack_modules__ = {
|
|
|
87
87
|
try {
|
|
88
88
|
this.isConnecting = true;
|
|
89
89
|
debugScrcpy('Starting scrcpy connection...');
|
|
90
|
-
const { AdbScrcpyClient
|
|
90
|
+
const { AdbScrcpyClient } = await import("@yume-chan/adb-scrcpy");
|
|
91
91
|
const { ReadableStream } = await import("@yume-chan/stream-extra");
|
|
92
92
|
const { DefaultServerPath } = await import("@yume-chan/scrcpy");
|
|
93
93
|
const serverBinPath = this.resolveServerBinPath();
|
|
94
94
|
await AdbScrcpyClient.pushServer(this.adb, ReadableStream.from((0, node_fs__rspack_import_0.createReadStream)(serverBinPath)));
|
|
95
|
-
const scrcpyOptions =
|
|
96
|
-
audio: false,
|
|
97
|
-
control: false,
|
|
98
|
-
maxSize: this.options.maxSize,
|
|
99
|
-
videoBitRate: this.options.videoBitRate,
|
|
100
|
-
maxFps: 10,
|
|
101
|
-
sendFrameMeta: true,
|
|
102
|
-
videoCodecOptions: 'i-frame-interval=0,bitrate-mode=2'
|
|
103
|
-
});
|
|
95
|
+
const scrcpyOptions = await this.createScrcpyOptions();
|
|
104
96
|
this.scrcpyClient = await AdbScrcpyClient.start(this.adb, DefaultServerPath, scrcpyOptions);
|
|
105
97
|
serverOutputTask = this.collectServerOutput(this.scrcpyClient.output, serverOutput);
|
|
106
98
|
const videoStreamPromise = this.scrcpyClient.videoStream;
|
|
@@ -128,6 +120,23 @@ var __webpack_modules__ = {
|
|
|
128
120
|
this.isConnecting = false;
|
|
129
121
|
}
|
|
130
122
|
}
|
|
123
|
+
async createScrcpyOptions() {
|
|
124
|
+
const [{ AdbScrcpyOptions3_3_3 }, { ScrcpyInstanceId }] = await Promise.all([
|
|
125
|
+
import("@yume-chan/adb-scrcpy"),
|
|
126
|
+
import("@yume-chan/scrcpy")
|
|
127
|
+
]);
|
|
128
|
+
return new AdbScrcpyOptions3_3_3({
|
|
129
|
+
audio: false,
|
|
130
|
+
control: false,
|
|
131
|
+
tunnelForward: true,
|
|
132
|
+
scid: ScrcpyInstanceId.random(),
|
|
133
|
+
maxSize: this.options.maxSize,
|
|
134
|
+
videoBitRate: this.options.videoBitRate,
|
|
135
|
+
maxFps: 10,
|
|
136
|
+
sendFrameMeta: true,
|
|
137
|
+
videoCodecOptions: 'i-frame-interval=0,bitrate-mode=2'
|
|
138
|
+
});
|
|
139
|
+
}
|
|
131
140
|
async collectServerOutput(output, lines) {
|
|
132
141
|
const reader = output.getReader();
|
|
133
142
|
try {
|
|
@@ -816,6 +825,230 @@ ${stdout ? truncateAdbShellStream(stdout, 'stdout') : EMPTY_ADB_SHELL_STDOUT}`;
|
|
|
816
825
|
this.retryAfter = null;
|
|
817
826
|
}
|
|
818
827
|
}
|
|
828
|
+
const TAG_RE = /<(\/?)([A-Za-z_][A-Za-z0-9_.\-:]*)([^>]*?)(\/?)>/g;
|
|
829
|
+
const ATTR_RE = /([A-Za-z_][A-Za-z0-9_.\-:]*)\s*=\s*("([^"]*)"|'([^']*)')/g;
|
|
830
|
+
const BOUNDS_RE = /^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$/;
|
|
831
|
+
const ENTITY_TABLE = {
|
|
832
|
+
amp: '&',
|
|
833
|
+
lt: '<',
|
|
834
|
+
gt: '>',
|
|
835
|
+
quot: '"',
|
|
836
|
+
apos: "'"
|
|
837
|
+
};
|
|
838
|
+
function decodeEntities(value) {
|
|
839
|
+
if (!value.includes('&')) return value;
|
|
840
|
+
return value.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (full, body)=>{
|
|
841
|
+
if (body.startsWith('#x') || body.startsWith('#X')) {
|
|
842
|
+
const code = Number.parseInt(body.slice(2), 16);
|
|
843
|
+
return Number.isFinite(code) ? String.fromCodePoint(code) : full;
|
|
844
|
+
}
|
|
845
|
+
if (body.startsWith('#')) {
|
|
846
|
+
const code = Number.parseInt(body.slice(1), 10);
|
|
847
|
+
return Number.isFinite(code) ? String.fromCodePoint(code) : full;
|
|
848
|
+
}
|
|
849
|
+
return ENTITY_TABLE[body] ?? full;
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
function parseAttrs(raw) {
|
|
853
|
+
const attrs = {};
|
|
854
|
+
ATTR_RE.lastIndex = 0;
|
|
855
|
+
let match;
|
|
856
|
+
while(match = ATTR_RE.exec(raw))attrs[match[1]] = decodeEntities(match[3] ?? match[4] ?? '');
|
|
857
|
+
return attrs;
|
|
858
|
+
}
|
|
859
|
+
function parseXml(input) {
|
|
860
|
+
if (0 === input.trim().length) throw new Error('parseAndroidUITreeXml: input is empty');
|
|
861
|
+
const cleaned = input.replace(/<\?xml[\s\S]*?\?>/g, '').replace(/<!DOCTYPE[\s\S]*?>/gi, '').replace(/<!--[\s\S]*?-->/g, '');
|
|
862
|
+
TAG_RE.lastIndex = 0;
|
|
863
|
+
const stack = [];
|
|
864
|
+
let root;
|
|
865
|
+
let match;
|
|
866
|
+
while(match = TAG_RE.exec(cleaned)){
|
|
867
|
+
const isClose = '/' === match[1];
|
|
868
|
+
const name = match[2];
|
|
869
|
+
if (isClose) {
|
|
870
|
+
const current = stack.pop();
|
|
871
|
+
if (!current || current.name !== name) throw new Error(`parseAndroidUITreeXml: unbalanced close tag </${name}>`);
|
|
872
|
+
continue;
|
|
873
|
+
}
|
|
874
|
+
const element = {
|
|
875
|
+
name,
|
|
876
|
+
attrs: parseAttrs(match[3] ?? ''),
|
|
877
|
+
children: []
|
|
878
|
+
};
|
|
879
|
+
const parent = stack[stack.length - 1];
|
|
880
|
+
if (parent) parent.children.push(element);
|
|
881
|
+
else if (root) throw new Error('parseAndroidUITreeXml: more than one top-level element');
|
|
882
|
+
else root = element;
|
|
883
|
+
if ('/' !== match[4]) stack.push(element);
|
|
884
|
+
}
|
|
885
|
+
if (stack.length > 0) throw new Error('parseAndroidUITreeXml: unclosed element');
|
|
886
|
+
if (!root) throw new Error('parseAndroidUITreeXml: no root element');
|
|
887
|
+
return root;
|
|
888
|
+
}
|
|
889
|
+
function parseBounds(raw, devicePixelRatio) {
|
|
890
|
+
const match = raw ? BOUNDS_RE.exec(raw) : void 0;
|
|
891
|
+
if (!match) return {
|
|
892
|
+
left: 0,
|
|
893
|
+
top: 0,
|
|
894
|
+
width: 0,
|
|
895
|
+
height: 0
|
|
896
|
+
};
|
|
897
|
+
const left = Number.parseInt(match[1], 10);
|
|
898
|
+
const top = Number.parseInt(match[2], 10);
|
|
899
|
+
const right = Number.parseInt(match[3], 10);
|
|
900
|
+
const bottom = Number.parseInt(match[4], 10);
|
|
901
|
+
const ratio = devicePixelRatio > 0 ? devicePixelRatio : 1;
|
|
902
|
+
return {
|
|
903
|
+
left: left / ratio,
|
|
904
|
+
top: top / ratio,
|
|
905
|
+
width: Math.max(0, right - left) / ratio,
|
|
906
|
+
height: Math.max(0, bottom - top) / ratio
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
function toUiNode(element, devicePixelRatio) {
|
|
910
|
+
const attrs = {};
|
|
911
|
+
for (const [name, value] of Object.entries(element.attrs))if ('bounds' !== name) attrs[name] = value;
|
|
912
|
+
return {
|
|
913
|
+
type: element.attrs.class || element.name,
|
|
914
|
+
attrs,
|
|
915
|
+
bounds: parseBounds(element.attrs.bounds, devicePixelRatio),
|
|
916
|
+
children: element.children.map((child)=>toUiNode(child, devicePixelRatio))
|
|
917
|
+
};
|
|
918
|
+
}
|
|
919
|
+
function uiautomatorXmlToUiNode(xml, devicePixelRatio) {
|
|
920
|
+
const root = parseXml(xml);
|
|
921
|
+
if ('hierarchy' === root.name && 1 === root.children.length) return toUiNode(root.children[0], devicePixelRatio);
|
|
922
|
+
return toUiNode(root, devicePixelRatio);
|
|
923
|
+
}
|
|
924
|
+
const ACCESSIBILITY_DUMP_ATTEMPTS = 3;
|
|
925
|
+
const ACCESSIBILITY_DUMP_TIMEOUT_MS = 5000;
|
|
926
|
+
const ACCESSIBILITY_DUMP_RETRY_DELAY_MS = 250;
|
|
927
|
+
const UIAUTOMATOR_LAYOUT_PATH = '/sdcard/midscene_window_dump.xml';
|
|
928
|
+
const debugUITree = (0, logger_.getDebug)('android:ui-tree');
|
|
929
|
+
function errorMessage(error) {
|
|
930
|
+
return error instanceof Error ? error.message : String(error);
|
|
931
|
+
}
|
|
932
|
+
function validateHierarchyXml(xml) {
|
|
933
|
+
if ('string' != typeof xml || !/<hierarchy(?:\s|>)/.test(xml) || !xml.includes('</hierarchy>')) throw new Error('uiautomator did not produce valid hierarchy XML');
|
|
934
|
+
}
|
|
935
|
+
function focusedPackageForLogicalDisplay(windowDump, logicalDisplayId) {
|
|
936
|
+
const displayMatches = [
|
|
937
|
+
...windowDump.matchAll(/^\s*Display: mDisplayId=(\d+)\b.*$/gm)
|
|
938
|
+
];
|
|
939
|
+
const displayIndex = displayMatches.findIndex((match)=>Number(match[1]) === logicalDisplayId);
|
|
940
|
+
if (-1 === displayIndex) return null;
|
|
941
|
+
const start = displayMatches[displayIndex].index ?? 0;
|
|
942
|
+
const end = displayMatches[displayIndex + 1]?.index ?? windowDump.length;
|
|
943
|
+
const displayBlock = windowDump.slice(start, end);
|
|
944
|
+
for (const field of [
|
|
945
|
+
'mCurrentFocus',
|
|
946
|
+
'mFocusedApp'
|
|
947
|
+
]){
|
|
948
|
+
const value = displayBlock.match(new RegExp(`^\\s*${field}=([^\\n]*)$`, 'm'))?.[1];
|
|
949
|
+
const packageName = value?.match(/\bu\d+\s+([A-Za-z0-9_$.-]+)\//)?.[1];
|
|
950
|
+
if (packageName) return packageName;
|
|
951
|
+
}
|
|
952
|
+
return null;
|
|
953
|
+
}
|
|
954
|
+
function firstTreePackage(root) {
|
|
955
|
+
if (root.attrs.package) return root.attrs.package;
|
|
956
|
+
for (const child of root.children){
|
|
957
|
+
const packageName = firstTreePackage(child);
|
|
958
|
+
if (packageName) return packageName;
|
|
959
|
+
}
|
|
960
|
+
return null;
|
|
961
|
+
}
|
|
962
|
+
function windowDisplayIdsForPackage(windowDump, packageName) {
|
|
963
|
+
const windowMatches = [
|
|
964
|
+
...windowDump.matchAll(/^\s*Window #\d+ Window\{.*$/gm)
|
|
965
|
+
];
|
|
966
|
+
const displayIds = new Set();
|
|
967
|
+
for(let index = 0; index < windowMatches.length; index++){
|
|
968
|
+
const start = windowMatches[index].index ?? 0;
|
|
969
|
+
const end = windowMatches[index + 1]?.index ?? windowDump.length;
|
|
970
|
+
const windowBlock = windowDump.slice(start, end);
|
|
971
|
+
const ownerPackage = windowBlock.match(/^\s*mOwnerUid=.*\bpackage=([A-Za-z0-9_$.-]+)\b/m)?.[1];
|
|
972
|
+
const headerPackage = windowMatches[index][0].match(/\bu\d+\s+([A-Za-z0-9_$.-]+)\//)?.[1];
|
|
973
|
+
if (ownerPackage !== packageName && headerPackage !== packageName) continue;
|
|
974
|
+
const displayId = windowBlock.match(/^\s*mDisplayId=(\d+)\b/m)?.[1];
|
|
975
|
+
if (void 0 !== displayId) displayIds.add(Number(displayId));
|
|
976
|
+
}
|
|
977
|
+
return [
|
|
978
|
+
...displayIds
|
|
979
|
+
];
|
|
980
|
+
}
|
|
981
|
+
function uiTreeExtent(root) {
|
|
982
|
+
let right = root.bounds.left + root.bounds.width;
|
|
983
|
+
let bottom = root.bounds.top + root.bounds.height;
|
|
984
|
+
for (const child of root.children){
|
|
985
|
+
const childExtent = uiTreeExtent(child);
|
|
986
|
+
right = Math.max(right, childExtent.right);
|
|
987
|
+
bottom = Math.max(bottom, childExtent.bottom);
|
|
988
|
+
}
|
|
989
|
+
return {
|
|
990
|
+
right,
|
|
991
|
+
bottom
|
|
992
|
+
};
|
|
993
|
+
}
|
|
994
|
+
async function dumpAccessibilityXml(adb) {
|
|
995
|
+
await runAdbShellStdoutOrThrow(adb, `rm -f ${UIAUTOMATOR_LAYOUT_PATH}`, {
|
|
996
|
+
timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
|
|
997
|
+
});
|
|
998
|
+
await runAdbShellStdoutOrThrow(adb, `uiautomator dump --compressed ${UIAUTOMATOR_LAYOUT_PATH}`, {
|
|
999
|
+
timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
|
|
1000
|
+
});
|
|
1001
|
+
const xml = await runAdbShellStdoutOrThrow(adb, `cat ${UIAUTOMATOR_LAYOUT_PATH}`, {
|
|
1002
|
+
timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
|
|
1003
|
+
});
|
|
1004
|
+
validateHierarchyXml(xml);
|
|
1005
|
+
return xml;
|
|
1006
|
+
}
|
|
1007
|
+
async function validateUITreeDisplay(root, options) {
|
|
1008
|
+
if ('number' != typeof options.displayId) return;
|
|
1009
|
+
const { adb, displayId } = options;
|
|
1010
|
+
const displayWindowDump = await runAdbShellStdoutOrThrow(adb, 'dumpsys window displays', {
|
|
1011
|
+
timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
|
|
1012
|
+
});
|
|
1013
|
+
const expectedPackage = focusedPackageForLogicalDisplay(displayWindowDump, displayId);
|
|
1014
|
+
const treePackage = firstTreePackage(root);
|
|
1015
|
+
if (!expectedPackage) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: no focused package found on the target display`);
|
|
1016
|
+
if (!treePackage) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: accessibility hierarchy has no package`);
|
|
1017
|
+
const allWindowDump = await runAdbShellStdoutOrThrow(adb, 'dumpsys window windows', {
|
|
1018
|
+
timeout: ACCESSIBILITY_DUMP_TIMEOUT_MS
|
|
1019
|
+
});
|
|
1020
|
+
const treePackageDisplayIds = windowDisplayIdsForPackage(allWindowDump, treePackage);
|
|
1021
|
+
if (0 === treePackageDisplayIds.length) throw new Error(`Unable to verify Android UI tree display alignment for displayId=${displayId}: no window display ownership found for accessibility package ${treePackage}`);
|
|
1022
|
+
if (!treePackageDisplayIds.includes(displayId)) throw new Error(`Android UI tree display mismatch for displayId=${displayId}: expected focused package ${expectedPackage}, but accessibility hierarchy belongs to ${treePackage} on displayId=${treePackageDisplayIds.join(',')}`);
|
|
1023
|
+
const targetSize = await options.getDisplaySize();
|
|
1024
|
+
const extent = uiTreeExtent(root);
|
|
1025
|
+
const tolerance = 1;
|
|
1026
|
+
if (extent.right > targetSize.width + tolerance || extent.bottom > targetSize.height + tolerance) throw new Error(`Android UI tree bounds exceed displayId=${displayId}: tree ${Math.round(extent.right)}x${Math.round(extent.bottom)} is outside target ${targetSize.width}x${targetSize.height}`);
|
|
1027
|
+
}
|
|
1028
|
+
async function captureAndroidUITree(options) {
|
|
1029
|
+
const failures = [];
|
|
1030
|
+
for(let attempt = 1; attempt <= ACCESSIBILITY_DUMP_ATTEMPTS; attempt++){
|
|
1031
|
+
const startedAt = Date.now();
|
|
1032
|
+
try {
|
|
1033
|
+
const xml = await dumpAccessibilityXml(options.adb);
|
|
1034
|
+
const capturedAt = Date.now();
|
|
1035
|
+
const root = uiautomatorXmlToUiNode(xml, options.devicePixelRatio);
|
|
1036
|
+
await validateUITreeDisplay(root, options);
|
|
1037
|
+
debugUITree('capture source=%s attempt=%d durationMs=%d bytes=%d', 'uiautomator', attempt, Date.now() - startedAt, xml.length);
|
|
1038
|
+
return {
|
|
1039
|
+
platform: 'android',
|
|
1040
|
+
capturedAt,
|
|
1041
|
+
root
|
|
1042
|
+
};
|
|
1043
|
+
} catch (error) {
|
|
1044
|
+
const failure = `uiautomator attempt ${attempt}/${ACCESSIBILITY_DUMP_ATTEMPTS}: ${errorMessage(error)}`;
|
|
1045
|
+
failures.push(failure);
|
|
1046
|
+
debugUITree('capture failed source=%s attempt=%d/%d durationMs=%d error=%s', 'uiautomator', attempt, ACCESSIBILITY_DUMP_ATTEMPTS, Date.now() - startedAt, errorMessage(error));
|
|
1047
|
+
if (attempt < ACCESSIBILITY_DUMP_ATTEMPTS) await (0, utils_namespaceObject.sleep)(ACCESSIBILITY_DUMP_RETRY_DELAY_MS);
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
throw new Error(`Unable to capture Android UI tree after ${failures.length} attempt(s): ${failures.join('; ')}`);
|
|
1051
|
+
}
|
|
819
1052
|
function device_define_property(obj, key, value) {
|
|
820
1053
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
821
1054
|
value: value,
|
|
@@ -835,6 +1068,35 @@ ${stdout ? truncateAdbShellStream(stdout, 'stdout') : EMPTY_ADB_SHELL_STDOUT}`;
|
|
|
835
1068
|
const warnDevice = (0, logger_.getDebug)('android:device', {
|
|
836
1069
|
console: true
|
|
837
1070
|
});
|
|
1071
|
+
function displayViewportForLogicalDisplay(displayDump, logicalDisplayId) {
|
|
1072
|
+
for (const viewportMatch of displayDump.matchAll(/DisplayViewport\{([^}]*)\}/g)){
|
|
1073
|
+
const viewport = viewportMatch[1];
|
|
1074
|
+
const displayId = viewport.match(/\bdisplayId=(\d+)\b/)?.[1];
|
|
1075
|
+
if (Number(displayId) === logicalDisplayId) return viewportMatch[0];
|
|
1076
|
+
}
|
|
1077
|
+
return null;
|
|
1078
|
+
}
|
|
1079
|
+
function displayInfoLineForPhysicalDisplay(displayDump, physicalDisplayId) {
|
|
1080
|
+
const lineRegex = new RegExp(`^.*uniqueId ["']local:${physicalDisplayId}["'].*$`, 'm');
|
|
1081
|
+
return displayDump.match(lineRegex)?.[0] ?? null;
|
|
1082
|
+
}
|
|
1083
|
+
function physicalDisplayIdForLogicalDisplay(displayDump, logicalDisplayId) {
|
|
1084
|
+
const viewport = displayViewportForLogicalDisplay(displayDump, logicalDisplayId);
|
|
1085
|
+
if (viewport) {
|
|
1086
|
+
const physicalId = viewport.match(/\buniqueId=['"]local:(\d+)['"]/)?.[1];
|
|
1087
|
+
if (physicalId) return physicalId;
|
|
1088
|
+
}
|
|
1089
|
+
const lines = displayDump.split(/\r?\n/);
|
|
1090
|
+
for(let index = 0; index < lines.length; index++){
|
|
1091
|
+
const displayId = lines[index].match(/^\s*mDisplayId=(\d+)\b/)?.[1];
|
|
1092
|
+
if (Number(displayId) === logicalDisplayId) for(let cursor = index + 1; cursor < lines.length; cursor++){
|
|
1093
|
+
if (/^\s*mDisplayId=\d+\b/.test(lines[cursor])) break;
|
|
1094
|
+
const physicalId = lines[cursor].match(/mBaseDisplayInfo=.*\buniqueId "local:(\d+)"/)?.[1];
|
|
1095
|
+
if (physicalId) return physicalId;
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
return null;
|
|
1099
|
+
}
|
|
838
1100
|
function escapeForShell(text) {
|
|
839
1101
|
return text.replace(/'/g, "'\\''").replace(/\n/g, '\\n');
|
|
840
1102
|
}
|
|
@@ -1122,6 +1384,15 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1122
1384
|
children: []
|
|
1123
1385
|
};
|
|
1124
1386
|
}
|
|
1387
|
+
async getUITree() {
|
|
1388
|
+
await this.initializeDevicePixelRatio();
|
|
1389
|
+
return captureAndroidUITree({
|
|
1390
|
+
adb: await this.getAdb(),
|
|
1391
|
+
devicePixelRatio: this.devicePixelRatio,
|
|
1392
|
+
displayId: this.options?.displayId,
|
|
1393
|
+
getDisplaySize: ()=>this.size()
|
|
1394
|
+
});
|
|
1395
|
+
}
|
|
1125
1396
|
async getScreenSize() {
|
|
1126
1397
|
const shouldCache = !(this.options?.alwaysRefreshScreenInfo ?? false);
|
|
1127
1398
|
debugDevice(`getScreenSize: alwaysRefreshScreenInfo=${this.options?.alwaysRefreshScreenInfo}, shouldCache=${shouldCache}, hasCachedSize=${!!this.cachedScreenSize}`);
|
|
@@ -1130,13 +1401,10 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1130
1401
|
if ('number' == typeof this.options?.displayId) try {
|
|
1131
1402
|
const stdout = await adb.shell('dumpsys display');
|
|
1132
1403
|
if (this.options?.usePhysicalDisplayIdForDisplayLookup) {
|
|
1133
|
-
const physicalDisplayId = await this.
|
|
1404
|
+
const physicalDisplayId = await this.resolvePhysicalDisplayId(stdout);
|
|
1134
1405
|
if (physicalDisplayId) {
|
|
1135
|
-
const
|
|
1136
|
-
|
|
1137
|
-
const lineMatch = stdout.match(lineRegex);
|
|
1138
|
-
if (lineMatch) {
|
|
1139
|
-
const targetLine = lineMatch[0];
|
|
1406
|
+
const targetLine = displayInfoLineForPhysicalDisplay(stdout, physicalDisplayId);
|
|
1407
|
+
if (targetLine) {
|
|
1140
1408
|
const realMatch = targetLine.match(/real (\d+) x (\d+)/);
|
|
1141
1409
|
const rotationMatch = targetLine.match(/rotation (\d+)/);
|
|
1142
1410
|
if (realMatch && rotationMatch) {
|
|
@@ -1157,10 +1425,8 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1157
1425
|
}
|
|
1158
1426
|
}
|
|
1159
1427
|
} else {
|
|
1160
|
-
const
|
|
1161
|
-
|
|
1162
|
-
if (match) {
|
|
1163
|
-
const targetLine = match[0];
|
|
1428
|
+
const targetLine = displayViewportForLogicalDisplay(stdout, this.options.displayId);
|
|
1429
|
+
if (targetLine) {
|
|
1164
1430
|
const physicalFrameMatch = targetLine.match(/physicalFrame=Rect\(\d+, \d+ - (\d+), (\d+)\)/);
|
|
1165
1431
|
const orientationMatch = targetLine.match(/orientation=(\d+)/);
|
|
1166
1432
|
if (physicalFrameMatch && orientationMatch) {
|
|
@@ -1226,13 +1492,10 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1226
1492
|
if ('number' == typeof this.options?.displayId) try {
|
|
1227
1493
|
const stdout = await adb.shell('dumpsys display');
|
|
1228
1494
|
if (this.options?.usePhysicalDisplayIdForDisplayLookup) {
|
|
1229
|
-
const physicalDisplayId = await this.
|
|
1495
|
+
const physicalDisplayId = await this.resolvePhysicalDisplayId(stdout);
|
|
1230
1496
|
if (physicalDisplayId) {
|
|
1231
|
-
const
|
|
1232
|
-
|
|
1233
|
-
const lineMatch = stdout.match(lineRegex);
|
|
1234
|
-
if (lineMatch) {
|
|
1235
|
-
const targetLine = lineMatch[0];
|
|
1497
|
+
const targetLine = displayInfoLineForPhysicalDisplay(stdout, physicalDisplayId);
|
|
1498
|
+
if (targetLine) {
|
|
1236
1499
|
const densityMatch = targetLine.match(/density (\d+)/);
|
|
1237
1500
|
if (densityMatch) {
|
|
1238
1501
|
const density = Number(densityMatch[1]);
|
|
@@ -1370,7 +1633,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1370
1633
|
debugDevice('screenshotBase64 end (scrcpy mode)');
|
|
1371
1634
|
return result;
|
|
1372
1635
|
} catch (error) {
|
|
1373
|
-
|
|
1636
|
+
warnDevice(`Scrcpy screenshot failed, falling back to standard ADB method.\nError: ${error}`);
|
|
1374
1637
|
}
|
|
1375
1638
|
const adb = await this.getAdb();
|
|
1376
1639
|
let screenshotBuffer;
|
|
@@ -1857,6 +2120,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1857
2120
|
if ('number' == typeof this.options?.displayId && 0 !== this.options.displayId) warnDevice(`yadb ${operation} cannot target display ${this.options.displayId}; it will act on the default display.`);
|
|
1858
2121
|
}
|
|
1859
2122
|
async getPhysicalDisplayId() {
|
|
2123
|
+
return this.resolvePhysicalDisplayId();
|
|
2124
|
+
}
|
|
2125
|
+
async resolvePhysicalDisplayId(displayDump) {
|
|
1860
2126
|
if (void 0 !== this.cachedPhysicalDisplayId) return this.cachedPhysicalDisplayId;
|
|
1861
2127
|
if ('number' != typeof this.options?.displayId) {
|
|
1862
2128
|
this.cachedPhysicalDisplayId = null;
|
|
@@ -1864,6 +2130,13 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
|
|
|
1864
2130
|
}
|
|
1865
2131
|
const adb = await this.getAdb();
|
|
1866
2132
|
try {
|
|
2133
|
+
const resolvedDisplayDump = displayDump ?? await adb.shell('dumpsys display');
|
|
2134
|
+
const logicalDisplayPhysicalId = physicalDisplayIdForLogicalDisplay(resolvedDisplayDump, this.options.displayId);
|
|
2135
|
+
if (logicalDisplayPhysicalId) {
|
|
2136
|
+
this.cachedPhysicalDisplayId = logicalDisplayPhysicalId;
|
|
2137
|
+
debugDevice(`Found and cached physical display ID: ${logicalDisplayPhysicalId} for logical display ID: ${this.options.displayId}`);
|
|
2138
|
+
return this.cachedPhysicalDisplayId;
|
|
2139
|
+
}
|
|
1867
2140
|
const stdout = await adb.shell(`dumpsys SurfaceFlinger --display-id ${this.options.displayId}`);
|
|
1868
2141
|
const regex = new RegExp(`Display (\\d+) \\(HWC display ${this.options.displayId}\\):`);
|
|
1869
2142
|
const displayMatch = stdout.match(regex);
|
package/dist/types/index.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ import { overrideAIConfig } from '@midscene/shared/env';
|
|
|
19
19
|
import { Point } from '@midscene/core';
|
|
20
20
|
import { Size } from '@midscene/core';
|
|
21
21
|
import type { ToolDefinition } from '@midscene/shared/agent-tools/types';
|
|
22
|
+
import { UITreeSnapshot } from '@midscene/core';
|
|
22
23
|
|
|
23
24
|
declare type ActionArgs<T extends DeviceAction> = [ActionParam<T>] extends [undefined] ? [] : [ActionParam<T>];
|
|
24
25
|
|
|
@@ -166,6 +167,7 @@ export declare class AndroidDevice implements AbstractInterface {
|
|
|
166
167
|
execYadb(keyboardContent: string): Promise<void>;
|
|
167
168
|
getElementsInfo(): Promise<ElementInfo[]>;
|
|
168
169
|
getElementsNodeTree(): Promise<any>;
|
|
170
|
+
getUITree(): Promise<UITreeSnapshot>;
|
|
169
171
|
getScreenSize(): Promise<{
|
|
170
172
|
override: string;
|
|
171
173
|
physical: string;
|
|
@@ -288,6 +290,7 @@ export declare class AndroidDevice implements AbstractInterface {
|
|
|
288
290
|
*/
|
|
289
291
|
private warnYadbOnNonDefaultDisplay;
|
|
290
292
|
getPhysicalDisplayId(): Promise<string | null>;
|
|
293
|
+
private resolvePhysicalDisplayId;
|
|
291
294
|
hideKeyboard(options?: AndroidDeviceInputOpt, timeoutMs?: number): Promise<boolean>;
|
|
292
295
|
}
|
|
293
296
|
|
|
@@ -396,8 +399,7 @@ export declare class ScrcpyDeviceAdapter {
|
|
|
396
399
|
* Resolve scrcpy config.
|
|
397
400
|
* maxSize defaults to 0 (no scaling, full physical resolution) so the Agent layer
|
|
398
401
|
* receives the highest quality image for AI processing.
|
|
399
|
-
* videoBitRate
|
|
400
|
-
* sufficient quality for all-I-frame H.264 encoding.
|
|
402
|
+
* videoBitRate uses the shared default unless explicitly configured.
|
|
401
403
|
*/
|
|
402
404
|
resolveConfig(deviceInfo: DevicePhysicalInfo): ResolvedScrcpyConfig;
|
|
403
405
|
/**
|
|
@@ -471,6 +473,7 @@ declare class ScrcpyScreenshotManager {
|
|
|
471
473
|
* Ensure scrcpy connection is active
|
|
472
474
|
*/
|
|
473
475
|
ensureConnected(): Promise<void>;
|
|
476
|
+
private createScrcpyOptions;
|
|
474
477
|
private collectServerOutput;
|
|
475
478
|
private createConnectionError;
|
|
476
479
|
private getErrorOutput;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/android",
|
|
3
|
-
"version": "1.10.
|
|
3
|
+
"version": "1.10.9-beta-20260731103801.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"type": "git",
|
|
6
6
|
"url": "https://github.com/web-infra-dev/midscene.git",
|
|
@@ -41,8 +41,8 @@
|
|
|
41
41
|
"@yume-chan/stream-extra": "2.1.0",
|
|
42
42
|
"appium-adb": "12.12.1",
|
|
43
43
|
"sharp": "^0.34.3",
|
|
44
|
-
"@midscene/
|
|
45
|
-
"@midscene/
|
|
44
|
+
"@midscene/core": "1.10.9-beta-20260731103801.0",
|
|
45
|
+
"@midscene/shared": "1.10.9-beta-20260731103801.0"
|
|
46
46
|
},
|
|
47
47
|
"optionalDependencies": {
|
|
48
48
|
"@ffmpeg-installer/ffmpeg": "^1.1.0"
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
"undici": "^6.0.0",
|
|
57
57
|
"vitest": "3.0.5",
|
|
58
58
|
"zod": "^3.25.1",
|
|
59
|
-
"@midscene/playground": "1.10.
|
|
59
|
+
"@midscene/playground": "1.10.9-beta-20260731103801.0"
|
|
60
60
|
},
|
|
61
61
|
"license": "MIT",
|
|
62
62
|
"scripts": {
|