@midscene/android 0.26.2 → 0.26.3-beta-20250813075706.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/index.mjs +711 -0
- package/dist/lib/index.js +750 -777
- package/dist/types/index.d.ts +114 -103
- package/package.json +11 -8
- package/dist/es/index.d.ts +0 -103
- package/dist/es/index.js +0 -770
- package/dist/lib/index.d.ts +0 -103
|
@@ -0,0 +1,711 @@
|
|
|
1
|
+
import node_assert from "node:assert";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import node_fs from "node:fs";
|
|
4
|
+
import node_path from "node:path";
|
|
5
|
+
import { getAIConfig } from "@midscene/core";
|
|
6
|
+
import { getTmpFile, sleep } from "@midscene/core/utils";
|
|
7
|
+
import { MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, overrideAIConfig, vlLocateMode } from "@midscene/shared/env";
|
|
8
|
+
import { isValidPNGImageBuffer, resizeImg } from "@midscene/shared/img";
|
|
9
|
+
import { getDebug } from "@midscene/shared/logger";
|
|
10
|
+
import { repeat } from "@midscene/shared/utils";
|
|
11
|
+
import { commonWebActions } from "@midscene/web";
|
|
12
|
+
import { ADB } from "appium-adb";
|
|
13
|
+
import { PageAgent } from "@midscene/web/agent";
|
|
14
|
+
function _define_property(obj, key, value) {
|
|
15
|
+
if (key in obj) Object.defineProperty(obj, key, {
|
|
16
|
+
value: value,
|
|
17
|
+
enumerable: true,
|
|
18
|
+
configurable: true,
|
|
19
|
+
writable: true
|
|
20
|
+
});
|
|
21
|
+
else obj[key] = value;
|
|
22
|
+
return obj;
|
|
23
|
+
}
|
|
24
|
+
const defaultScrollUntilTimes = 10;
|
|
25
|
+
const defaultFastScrollDuration = 100;
|
|
26
|
+
const defaultNormalScrollDuration = 1000;
|
|
27
|
+
const debugPage = getDebug('android:device');
|
|
28
|
+
const asyncNoop = async ()=>{};
|
|
29
|
+
const androidActions = [
|
|
30
|
+
{
|
|
31
|
+
name: 'AndroidBackButton',
|
|
32
|
+
description: 'Trigger the system "back" operation on Android devices',
|
|
33
|
+
location: false,
|
|
34
|
+
call: asyncNoop
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
name: 'AndroidHomeButton',
|
|
38
|
+
description: 'Trigger the system "home" operation on Android devices',
|
|
39
|
+
location: false,
|
|
40
|
+
call: asyncNoop
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
name: 'AndroidRecentAppsButton',
|
|
44
|
+
description: 'Trigger the system "recent apps" operation on Android devices',
|
|
45
|
+
location: false,
|
|
46
|
+
call: asyncNoop
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
name: 'AndroidLongPress',
|
|
50
|
+
description: 'Trigger a long press on the screen at specified coordinates on Android devices',
|
|
51
|
+
paramSchema: '{ duration?: number }',
|
|
52
|
+
paramDescription: 'The duration of the long press',
|
|
53
|
+
location: 'optional',
|
|
54
|
+
whatToLocate: 'The element to be long pressed',
|
|
55
|
+
call: asyncNoop
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
name: 'AndroidPull',
|
|
59
|
+
description: 'Trigger pull down to refresh or pull up actions on Android devices',
|
|
60
|
+
paramSchema: '{ direction: "up" | "down", distance?: number, duration?: number }',
|
|
61
|
+
paramDescription: 'The direction to pull, the distance to pull, and the duration of the pull.',
|
|
62
|
+
location: 'optional',
|
|
63
|
+
whatToLocate: 'The element to be pulled',
|
|
64
|
+
call: asyncNoop
|
|
65
|
+
}
|
|
66
|
+
];
|
|
67
|
+
class AndroidDevice {
|
|
68
|
+
actionSpace() {
|
|
69
|
+
return commonWebActions.concat(androidActions);
|
|
70
|
+
}
|
|
71
|
+
async connect() {
|
|
72
|
+
return this.getAdb();
|
|
73
|
+
}
|
|
74
|
+
async getAdb() {
|
|
75
|
+
if (this.destroyed) throw new Error(`AndroidDevice ${this.deviceId} has been destroyed and cannot execute ADB commands`);
|
|
76
|
+
if (this.adb) return this.createAdbProxy(this.adb);
|
|
77
|
+
if (this.connectingAdb) return this.connectingAdb.then((adb)=>this.createAdbProxy(adb));
|
|
78
|
+
this.connectingAdb = (async ()=>{
|
|
79
|
+
let error = null;
|
|
80
|
+
debugPage(`Initializing ADB with device ID: ${this.deviceId}`);
|
|
81
|
+
try {
|
|
82
|
+
var _this_options, _this_options1, _this_options2;
|
|
83
|
+
const androidAdbPath = (null == (_this_options = this.options) ? void 0 : _this_options.androidAdbPath) || getAIConfig(MIDSCENE_ADB_PATH);
|
|
84
|
+
const remoteAdbHost = (null == (_this_options1 = this.options) ? void 0 : _this_options1.remoteAdbHost) || getAIConfig(MIDSCENE_ADB_REMOTE_HOST);
|
|
85
|
+
const remoteAdbPort = (null == (_this_options2 = this.options) ? void 0 : _this_options2.remoteAdbPort) || getAIConfig(MIDSCENE_ADB_REMOTE_PORT);
|
|
86
|
+
this.adb = await new ADB({
|
|
87
|
+
udid: this.deviceId,
|
|
88
|
+
adbExecTimeout: 60000,
|
|
89
|
+
executable: androidAdbPath ? {
|
|
90
|
+
path: androidAdbPath,
|
|
91
|
+
defaultArgs: []
|
|
92
|
+
} : void 0,
|
|
93
|
+
remoteAdbHost: remoteAdbHost || void 0,
|
|
94
|
+
remoteAdbPort: remoteAdbPort ? Number(remoteAdbPort) : void 0
|
|
95
|
+
});
|
|
96
|
+
const size = await this.getScreenSize();
|
|
97
|
+
console.log(`
|
|
98
|
+
DeviceId: ${this.deviceId}
|
|
99
|
+
ScreenSize:
|
|
100
|
+
${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[key]}${'override' === key && size[key] ? " \u2705" : ''}`).join('\n')}
|
|
101
|
+
`);
|
|
102
|
+
debugPage('ADB initialized successfully');
|
|
103
|
+
return this.adb;
|
|
104
|
+
} catch (e) {
|
|
105
|
+
debugPage(`Failed to initialize ADB: ${e}`);
|
|
106
|
+
error = new Error(`Unable to connect to device ${this.deviceId}: ${e}`);
|
|
107
|
+
} finally{
|
|
108
|
+
this.connectingAdb = null;
|
|
109
|
+
}
|
|
110
|
+
if (error) throw error;
|
|
111
|
+
throw new Error('ADB initialization failed unexpectedly');
|
|
112
|
+
})();
|
|
113
|
+
return this.connectingAdb;
|
|
114
|
+
}
|
|
115
|
+
createAdbProxy(adb) {
|
|
116
|
+
return new Proxy(adb, {
|
|
117
|
+
get: (target, prop)=>{
|
|
118
|
+
const originalMethod = target[prop];
|
|
119
|
+
if ('function' != typeof originalMethod) return originalMethod;
|
|
120
|
+
return async (...args)=>{
|
|
121
|
+
try {
|
|
122
|
+
debugPage(`adb ${String(prop)} ${args.join(' ')}`);
|
|
123
|
+
const result = await originalMethod.apply(target, args);
|
|
124
|
+
debugPage(`adb ${String(prop)} ${args.join(' ')} end`);
|
|
125
|
+
return result;
|
|
126
|
+
} catch (error) {
|
|
127
|
+
const methodName = String(prop);
|
|
128
|
+
const deviceId = this.deviceId;
|
|
129
|
+
debugPage(`ADB error with device ${deviceId} when calling ${methodName}: ${error}`);
|
|
130
|
+
throw new Error(`ADB error with device ${deviceId} when calling ${methodName}, please check https://midscenejs.com/integrate-with-android.html#faq : ${error.message}`, {
|
|
131
|
+
cause: error
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
async launch(uri) {
|
|
139
|
+
const adb = await this.getAdb();
|
|
140
|
+
this.uri = uri;
|
|
141
|
+
try {
|
|
142
|
+
debugPage(`Launching app: ${uri}`);
|
|
143
|
+
if (uri.startsWith('http://') || uri.startsWith('https://') || uri.includes('://')) await adb.startUri(uri);
|
|
144
|
+
else if (uri.includes('/')) {
|
|
145
|
+
const [appPackage, appActivity] = uri.split('/');
|
|
146
|
+
await adb.startApp({
|
|
147
|
+
pkg: appPackage,
|
|
148
|
+
activity: appActivity
|
|
149
|
+
});
|
|
150
|
+
} else await adb.activateApp(uri);
|
|
151
|
+
debugPage(`Successfully launched: ${uri}`);
|
|
152
|
+
} catch (error) {
|
|
153
|
+
debugPage(`Error launching ${uri}: ${error}`);
|
|
154
|
+
throw new Error(`Failed to launch ${uri}: ${error.message}`, {
|
|
155
|
+
cause: error
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
return this;
|
|
159
|
+
}
|
|
160
|
+
async execYadb(keyboardContent) {
|
|
161
|
+
await this.ensureYadb();
|
|
162
|
+
const adb = await this.getAdb();
|
|
163
|
+
await adb.shell(`app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "${keyboardContent}"`);
|
|
164
|
+
}
|
|
165
|
+
async getElementsInfo() {
|
|
166
|
+
return [];
|
|
167
|
+
}
|
|
168
|
+
async getElementsNodeTree() {
|
|
169
|
+
return {
|
|
170
|
+
node: null,
|
|
171
|
+
children: []
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
async getScreenSize() {
|
|
175
|
+
const adb = await this.getAdb();
|
|
176
|
+
const stdout = await adb.shell([
|
|
177
|
+
'wm',
|
|
178
|
+
'size'
|
|
179
|
+
]);
|
|
180
|
+
const size = {
|
|
181
|
+
override: '',
|
|
182
|
+
physical: ''
|
|
183
|
+
};
|
|
184
|
+
const overrideSize = new RegExp(/Override size: ([^\r?\n]+)*/g).exec(stdout);
|
|
185
|
+
if (overrideSize && overrideSize.length >= 2 && overrideSize[1]) {
|
|
186
|
+
debugPage(`Using Override size: ${overrideSize[1].trim()}`);
|
|
187
|
+
size.override = overrideSize[1].trim();
|
|
188
|
+
}
|
|
189
|
+
const physicalSize = new RegExp(/Physical size: ([^\r?\n]+)*/g).exec(stdout);
|
|
190
|
+
if (physicalSize && physicalSize.length >= 2) {
|
|
191
|
+
debugPage(`Using Physical size: ${physicalSize[1].trim()}`);
|
|
192
|
+
size.physical = physicalSize[1].trim();
|
|
193
|
+
}
|
|
194
|
+
let orientation = 0;
|
|
195
|
+
try {
|
|
196
|
+
const orientationStdout = await adb.shell('dumpsys input | grep SurfaceOrientation');
|
|
197
|
+
const orientationMatch = orientationStdout.match(/SurfaceOrientation:\s*(\d)/);
|
|
198
|
+
if (!orientationMatch) throw new Error('Failed to get orientation from input');
|
|
199
|
+
orientation = Number(orientationMatch[1]);
|
|
200
|
+
debugPage(`Screen orientation: ${orientation}`);
|
|
201
|
+
} catch (e) {
|
|
202
|
+
debugPage('Failed to get orientation from input, try display');
|
|
203
|
+
try {
|
|
204
|
+
const orientationStdout = await adb.shell('dumpsys display | grep mCurrentOrientation');
|
|
205
|
+
const orientationMatch = orientationStdout.match(/mCurrentOrientation=(\d)/);
|
|
206
|
+
if (!orientationMatch) throw new Error('Failed to get orientation from display');
|
|
207
|
+
orientation = Number(orientationMatch[1]);
|
|
208
|
+
debugPage(`Screen orientation (fallback): ${orientation}`);
|
|
209
|
+
} catch (e2) {
|
|
210
|
+
orientation = 0;
|
|
211
|
+
debugPage('Failed to get orientation from display, default to 0');
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
if (size.override || size.physical) return {
|
|
215
|
+
...size,
|
|
216
|
+
orientation
|
|
217
|
+
};
|
|
218
|
+
throw new Error(`Failed to get screen size, output: ${stdout}`);
|
|
219
|
+
}
|
|
220
|
+
async size() {
|
|
221
|
+
const adb = await this.getAdb();
|
|
222
|
+
const screenSize = await this.getScreenSize();
|
|
223
|
+
const match = (screenSize.override || screenSize.physical).match(/(\d+)x(\d+)/);
|
|
224
|
+
if (!match || match.length < 3) throw new Error(`Unable to parse screen size: ${screenSize}`);
|
|
225
|
+
const isLandscape = 1 === screenSize.orientation || 3 === screenSize.orientation;
|
|
226
|
+
const width = Number.parseInt(match[isLandscape ? 2 : 1], 10);
|
|
227
|
+
const height = Number.parseInt(match[isLandscape ? 1 : 2], 10);
|
|
228
|
+
const densityNum = await adb.getScreenDensity();
|
|
229
|
+
this.devicePixelRatio = Number(densityNum) / 160;
|
|
230
|
+
const { x: logicalWidth, y: logicalHeight } = this.reverseAdjustCoordinates(width, height);
|
|
231
|
+
return {
|
|
232
|
+
width: logicalWidth,
|
|
233
|
+
height: logicalHeight,
|
|
234
|
+
dpr: this.devicePixelRatio
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
adjustCoordinates(x, y) {
|
|
238
|
+
const ratio = this.devicePixelRatio;
|
|
239
|
+
return {
|
|
240
|
+
x: Math.round(x * ratio),
|
|
241
|
+
y: Math.round(y * ratio)
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
reverseAdjustCoordinates(x, y) {
|
|
245
|
+
const ratio = this.devicePixelRatio;
|
|
246
|
+
return {
|
|
247
|
+
x: Math.round(x / ratio),
|
|
248
|
+
y: Math.round(y / ratio)
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
async screenshotBase64() {
|
|
252
|
+
debugPage('screenshotBase64 begin');
|
|
253
|
+
const { width, height } = await this.size();
|
|
254
|
+
const adb = await this.getAdb();
|
|
255
|
+
let screenshotBuffer;
|
|
256
|
+
const androidScreenshotPath = `/data/local/tmp/midscene_screenshot_${randomUUID()}.png`;
|
|
257
|
+
try {
|
|
258
|
+
debugPage('Taking screenshot via adb.takeScreenshot');
|
|
259
|
+
screenshotBuffer = await adb.takeScreenshot(null);
|
|
260
|
+
debugPage('adb.takeScreenshot completed');
|
|
261
|
+
if (!screenshotBuffer) throw new Error('Failed to capture screenshot: screenshotBuffer is null');
|
|
262
|
+
if (!isValidPNGImageBuffer(screenshotBuffer)) {
|
|
263
|
+
debugPage('Invalid image buffer detected: not a valid image format');
|
|
264
|
+
throw new Error('Screenshot buffer has invalid format: could not find valid image signature');
|
|
265
|
+
}
|
|
266
|
+
} catch (error) {
|
|
267
|
+
const screenshotPath = getTmpFile('png');
|
|
268
|
+
try {
|
|
269
|
+
debugPage('Fallback: taking screenshot via shell screencap');
|
|
270
|
+
try {
|
|
271
|
+
await adb.shell(`screencap -p ${androidScreenshotPath}`);
|
|
272
|
+
debugPage('adb.shell screencap completed');
|
|
273
|
+
} catch (error) {
|
|
274
|
+
debugPage('screencap failed, using forceScreenshot');
|
|
275
|
+
await this.forceScreenshot(androidScreenshotPath);
|
|
276
|
+
debugPage('forceScreenshot completed');
|
|
277
|
+
}
|
|
278
|
+
debugPage('Pulling screenshot file from device');
|
|
279
|
+
await adb.pull(androidScreenshotPath, screenshotPath);
|
|
280
|
+
debugPage('adb.pull completed');
|
|
281
|
+
screenshotBuffer = await node_fs.promises.readFile(screenshotPath);
|
|
282
|
+
} finally{
|
|
283
|
+
await adb.shell(`rm -f ${androidScreenshotPath}`);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
debugPage('Resizing screenshot image');
|
|
287
|
+
const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, {
|
|
288
|
+
width,
|
|
289
|
+
height
|
|
290
|
+
});
|
|
291
|
+
debugPage('Image resize completed');
|
|
292
|
+
debugPage('Converting to base64');
|
|
293
|
+
const result = `data:image/jpeg;base64,${resizedScreenshotBuffer.toString('base64')}`;
|
|
294
|
+
debugPage('screenshotBase64 end');
|
|
295
|
+
return result;
|
|
296
|
+
}
|
|
297
|
+
get mouse() {
|
|
298
|
+
return {
|
|
299
|
+
click: (x, y)=>this.mouseClick(x, y),
|
|
300
|
+
wheel: (deltaX, deltaY)=>this.mouseWheel(deltaX, deltaY),
|
|
301
|
+
move: (x, y)=>this.mouseMove(x, y),
|
|
302
|
+
drag: (from, to)=>this.mouseDrag(from, to)
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
get keyboard() {
|
|
306
|
+
return {
|
|
307
|
+
type: (text, options)=>this.keyboardType(text, options),
|
|
308
|
+
press: (action)=>this.keyboardPressAction(action)
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
async clearInput(element) {
|
|
312
|
+
if (!element) return;
|
|
313
|
+
await this.ensureYadb();
|
|
314
|
+
const adb = await this.getAdb();
|
|
315
|
+
await this.mouse.click(element.center[0], element.center[1]);
|
|
316
|
+
await adb.shell('app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "~CLEAR~"');
|
|
317
|
+
if (await adb.isSoftKeyboardPresent()) return;
|
|
318
|
+
await this.mouse.click(element.center[0], element.center[1]);
|
|
319
|
+
}
|
|
320
|
+
async forceScreenshot(path) {
|
|
321
|
+
await this.ensureYadb();
|
|
322
|
+
const adb = await this.getAdb();
|
|
323
|
+
await adb.shell(`app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -screenshot ${path}`);
|
|
324
|
+
}
|
|
325
|
+
async url() {
|
|
326
|
+
return '';
|
|
327
|
+
}
|
|
328
|
+
async scrollUntilTop(startPoint) {
|
|
329
|
+
if (startPoint) {
|
|
330
|
+
const start = {
|
|
331
|
+
x: startPoint.left,
|
|
332
|
+
y: startPoint.top
|
|
333
|
+
};
|
|
334
|
+
const end = {
|
|
335
|
+
x: start.x,
|
|
336
|
+
y: 0
|
|
337
|
+
};
|
|
338
|
+
await this.mouseDrag(start, end);
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
await repeat(defaultScrollUntilTimes, ()=>this.mouseWheel(0, 9999999, defaultFastScrollDuration));
|
|
342
|
+
await sleep(1000);
|
|
343
|
+
}
|
|
344
|
+
async scrollUntilBottom(startPoint) {
|
|
345
|
+
if (startPoint) {
|
|
346
|
+
const { height } = await this.size();
|
|
347
|
+
const start = {
|
|
348
|
+
x: startPoint.left,
|
|
349
|
+
y: startPoint.top
|
|
350
|
+
};
|
|
351
|
+
const end = {
|
|
352
|
+
x: start.x,
|
|
353
|
+
y: height
|
|
354
|
+
};
|
|
355
|
+
await this.mouseDrag(start, end);
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
await repeat(defaultScrollUntilTimes, ()=>this.mouseWheel(0, -9999999, defaultFastScrollDuration));
|
|
359
|
+
await sleep(1000);
|
|
360
|
+
}
|
|
361
|
+
async scrollUntilLeft(startPoint) {
|
|
362
|
+
if (startPoint) {
|
|
363
|
+
const start = {
|
|
364
|
+
x: startPoint.left,
|
|
365
|
+
y: startPoint.top
|
|
366
|
+
};
|
|
367
|
+
const end = {
|
|
368
|
+
x: 0,
|
|
369
|
+
y: start.y
|
|
370
|
+
};
|
|
371
|
+
await this.mouseDrag(start, end);
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
await repeat(defaultScrollUntilTimes, ()=>this.mouseWheel(9999999, 0, defaultFastScrollDuration));
|
|
375
|
+
await sleep(1000);
|
|
376
|
+
}
|
|
377
|
+
async scrollUntilRight(startPoint) {
|
|
378
|
+
if (startPoint) {
|
|
379
|
+
const { width } = await this.size();
|
|
380
|
+
const start = {
|
|
381
|
+
x: startPoint.left,
|
|
382
|
+
y: startPoint.top
|
|
383
|
+
};
|
|
384
|
+
const end = {
|
|
385
|
+
x: width,
|
|
386
|
+
y: start.y
|
|
387
|
+
};
|
|
388
|
+
await this.mouseDrag(start, end);
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
await repeat(defaultScrollUntilTimes, ()=>this.mouseWheel(-9999999, 0, defaultFastScrollDuration));
|
|
392
|
+
await sleep(1000);
|
|
393
|
+
}
|
|
394
|
+
async scrollUp(distance, startPoint) {
|
|
395
|
+
const { height } = await this.size();
|
|
396
|
+
const scrollDistance = distance || height;
|
|
397
|
+
if (startPoint) {
|
|
398
|
+
const start = {
|
|
399
|
+
x: startPoint.left,
|
|
400
|
+
y: startPoint.top
|
|
401
|
+
};
|
|
402
|
+
const endY = Math.max(0, start.y - scrollDistance);
|
|
403
|
+
const end = {
|
|
404
|
+
x: start.x,
|
|
405
|
+
y: endY
|
|
406
|
+
};
|
|
407
|
+
await this.mouseDrag(start, end);
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
await this.mouseWheel(0, scrollDistance);
|
|
411
|
+
}
|
|
412
|
+
async scrollDown(distance, startPoint) {
|
|
413
|
+
const { height } = await this.size();
|
|
414
|
+
const scrollDistance = distance || height;
|
|
415
|
+
if (startPoint) {
|
|
416
|
+
const start = {
|
|
417
|
+
x: startPoint.left,
|
|
418
|
+
y: startPoint.top
|
|
419
|
+
};
|
|
420
|
+
const endY = Math.min(height, start.y + scrollDistance);
|
|
421
|
+
const end = {
|
|
422
|
+
x: start.x,
|
|
423
|
+
y: endY
|
|
424
|
+
};
|
|
425
|
+
await this.mouseDrag(start, end);
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
await this.mouseWheel(0, -scrollDistance);
|
|
429
|
+
}
|
|
430
|
+
async scrollLeft(distance, startPoint) {
|
|
431
|
+
const { width } = await this.size();
|
|
432
|
+
const scrollDistance = distance || width;
|
|
433
|
+
if (startPoint) {
|
|
434
|
+
const start = {
|
|
435
|
+
x: startPoint.left,
|
|
436
|
+
y: startPoint.top
|
|
437
|
+
};
|
|
438
|
+
const endX = Math.max(0, start.x - scrollDistance);
|
|
439
|
+
const end = {
|
|
440
|
+
x: endX,
|
|
441
|
+
y: start.y
|
|
442
|
+
};
|
|
443
|
+
await this.mouseDrag(start, end);
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
await this.mouseWheel(scrollDistance, 0);
|
|
447
|
+
}
|
|
448
|
+
async scrollRight(distance, startPoint) {
|
|
449
|
+
const { width } = await this.size();
|
|
450
|
+
const scrollDistance = distance || width;
|
|
451
|
+
if (startPoint) {
|
|
452
|
+
const start = {
|
|
453
|
+
x: startPoint.left,
|
|
454
|
+
y: startPoint.top
|
|
455
|
+
};
|
|
456
|
+
const endX = Math.min(width, start.x + scrollDistance);
|
|
457
|
+
const end = {
|
|
458
|
+
x: endX,
|
|
459
|
+
y: start.y
|
|
460
|
+
};
|
|
461
|
+
await this.mouseDrag(start, end);
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
await this.mouseWheel(-scrollDistance, 0);
|
|
465
|
+
}
|
|
466
|
+
async ensureYadb() {
|
|
467
|
+
if (!this.yadbPushed) {
|
|
468
|
+
const adb = await this.getAdb();
|
|
469
|
+
const androidPkgJson = require.resolve('@midscene/android/package.json');
|
|
470
|
+
const yadbBin = node_path.join(node_path.dirname(androidPkgJson), 'bin', 'yadb');
|
|
471
|
+
await adb.push(yadbBin, '/data/local/tmp');
|
|
472
|
+
this.yadbPushed = true;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
async keyboardType(text, options) {
|
|
476
|
+
var _this_options, _this_options1;
|
|
477
|
+
if (!text) return;
|
|
478
|
+
const adb = await this.getAdb();
|
|
479
|
+
const isChinese = /[\p{Script=Han}\p{sc=Hani}]/u.test(text);
|
|
480
|
+
const IME_STRATEGY = ((null == (_this_options = this.options) ? void 0 : _this_options.imeStrategy) || getAIConfig(MIDSCENE_ANDROID_IME_STRATEGY)) ?? 'always-yadb';
|
|
481
|
+
const isAutoDismissKeyboard = (null == options ? void 0 : options.autoDismissKeyboard) ?? (null == (_this_options1 = this.options) ? void 0 : _this_options1.autoDismissKeyboard) ?? true;
|
|
482
|
+
if ('always-yadb' === IME_STRATEGY || 'yadb-for-non-ascii' === IME_STRATEGY && isChinese) await this.execYadb(text);
|
|
483
|
+
else await adb.inputText(text);
|
|
484
|
+
if (true === isAutoDismissKeyboard) await this.hideKeyboard(options);
|
|
485
|
+
}
|
|
486
|
+
async keyboardPress(key) {
|
|
487
|
+
const keyCodeMap = {
|
|
488
|
+
Enter: 66,
|
|
489
|
+
Backspace: 67,
|
|
490
|
+
Tab: 61,
|
|
491
|
+
ArrowUp: 19,
|
|
492
|
+
ArrowDown: 20,
|
|
493
|
+
ArrowLeft: 21,
|
|
494
|
+
ArrowRight: 22,
|
|
495
|
+
Escape: 111,
|
|
496
|
+
Home: 3,
|
|
497
|
+
End: 123
|
|
498
|
+
};
|
|
499
|
+
const adb = await this.getAdb();
|
|
500
|
+
const keyCode = keyCodeMap[key];
|
|
501
|
+
if (void 0 !== keyCode) await adb.keyevent(keyCode);
|
|
502
|
+
else if (1 === key.length) {
|
|
503
|
+
const asciiCode = key.toUpperCase().charCodeAt(0);
|
|
504
|
+
if (asciiCode >= 65 && asciiCode <= 90) await adb.keyevent(asciiCode - 36);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
async keyboardPressAction(action) {
|
|
508
|
+
if (Array.isArray(action)) for (const act of action)await this.keyboardPress(act.key);
|
|
509
|
+
else await this.keyboardPress(action.key);
|
|
510
|
+
}
|
|
511
|
+
async mouseClick(x, y) {
|
|
512
|
+
const adb = await this.getAdb();
|
|
513
|
+
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
514
|
+
await adb.shell(`input swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} 150`);
|
|
515
|
+
}
|
|
516
|
+
async mouseMove(x, y) {
|
|
517
|
+
return Promise.resolve();
|
|
518
|
+
}
|
|
519
|
+
async mouseDrag(from, to) {
|
|
520
|
+
const adb = await this.getAdb();
|
|
521
|
+
const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
|
|
522
|
+
const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
|
|
523
|
+
await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} 300`);
|
|
524
|
+
}
|
|
525
|
+
async mouseWheel(deltaX, deltaY, duration = defaultNormalScrollDuration) {
|
|
526
|
+
const { width, height } = await this.size();
|
|
527
|
+
const n = 4;
|
|
528
|
+
const startX = deltaX < 0 ? width / n * (n - 1) : width / n;
|
|
529
|
+
const startY = deltaY < 0 ? height / n * (n - 1) : height / n;
|
|
530
|
+
const maxNegativeDeltaX = startX;
|
|
531
|
+
const maxPositiveDeltaX = width / n * (n - 1);
|
|
532
|
+
const maxNegativeDeltaY = startY;
|
|
533
|
+
const maxPositiveDeltaY = height / n * (n - 1);
|
|
534
|
+
deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
|
|
535
|
+
deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
|
|
536
|
+
const endX = startX + deltaX;
|
|
537
|
+
const endY = startY + deltaY;
|
|
538
|
+
const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(startX, startY);
|
|
539
|
+
const { x: adjustedEndX, y: adjustedEndY } = this.adjustCoordinates(endX, endY);
|
|
540
|
+
const adb = await this.getAdb();
|
|
541
|
+
await adb.shell(`input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${duration}`);
|
|
542
|
+
}
|
|
543
|
+
async destroy() {
|
|
544
|
+
if (this.destroyed) return;
|
|
545
|
+
this.destroyed = true;
|
|
546
|
+
try {
|
|
547
|
+
if (this.adb) this.adb = null;
|
|
548
|
+
} catch (error) {
|
|
549
|
+
console.error('Error during cleanup:', error);
|
|
550
|
+
}
|
|
551
|
+
this.connectingAdb = null;
|
|
552
|
+
this.yadbPushed = false;
|
|
553
|
+
}
|
|
554
|
+
async back() {
|
|
555
|
+
const adb = await this.getAdb();
|
|
556
|
+
await adb.shell('input keyevent 4');
|
|
557
|
+
}
|
|
558
|
+
async home() {
|
|
559
|
+
const adb = await this.getAdb();
|
|
560
|
+
await adb.shell('input keyevent 3');
|
|
561
|
+
}
|
|
562
|
+
async recentApps() {
|
|
563
|
+
const adb = await this.getAdb();
|
|
564
|
+
await adb.shell('input keyevent 187');
|
|
565
|
+
}
|
|
566
|
+
async longPress(x, y, duration = 1000) {
|
|
567
|
+
const adb = await this.getAdb();
|
|
568
|
+
const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
|
|
569
|
+
await adb.shell(`input swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} ${duration}`);
|
|
570
|
+
}
|
|
571
|
+
async pullDown(startPoint, distance, duration = 800) {
|
|
572
|
+
const { width, height } = await this.size();
|
|
573
|
+
const start = startPoint ? {
|
|
574
|
+
x: startPoint.left,
|
|
575
|
+
y: startPoint.top
|
|
576
|
+
} : {
|
|
577
|
+
x: width / 2,
|
|
578
|
+
y: 0.15 * height
|
|
579
|
+
};
|
|
580
|
+
const pullDistance = distance || 0.5 * height;
|
|
581
|
+
const end = {
|
|
582
|
+
x: start.x,
|
|
583
|
+
y: start.y + pullDistance
|
|
584
|
+
};
|
|
585
|
+
await this.pullDrag(start, end, duration);
|
|
586
|
+
await sleep(200);
|
|
587
|
+
}
|
|
588
|
+
async pullDrag(from, to, duration) {
|
|
589
|
+
const adb = await this.getAdb();
|
|
590
|
+
const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
|
|
591
|
+
const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
|
|
592
|
+
await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} ${duration}`);
|
|
593
|
+
}
|
|
594
|
+
async pullUp(startPoint, distance, duration = 600) {
|
|
595
|
+
const { width, height } = await this.size();
|
|
596
|
+
const start = startPoint ? {
|
|
597
|
+
x: startPoint.left,
|
|
598
|
+
y: startPoint.top
|
|
599
|
+
} : {
|
|
600
|
+
x: width / 2,
|
|
601
|
+
y: 0.85 * height
|
|
602
|
+
};
|
|
603
|
+
const pullDistance = distance || 0.4 * height;
|
|
604
|
+
const end = {
|
|
605
|
+
x: start.x,
|
|
606
|
+
y: start.y - pullDistance
|
|
607
|
+
};
|
|
608
|
+
await this.pullDrag(start, end, duration);
|
|
609
|
+
await sleep(100);
|
|
610
|
+
}
|
|
611
|
+
async getXpathsById(id) {
|
|
612
|
+
throw new Error('Not implemented');
|
|
613
|
+
}
|
|
614
|
+
async getXpathsByPoint(point, isOrderSensitive) {
|
|
615
|
+
throw new Error('Not implemented');
|
|
616
|
+
}
|
|
617
|
+
async getElementInfoByXpath(xpath) {
|
|
618
|
+
throw new Error('Not implemented');
|
|
619
|
+
}
|
|
620
|
+
async hideKeyboard(options, timeoutMs = 1000) {
|
|
621
|
+
var _this_options;
|
|
622
|
+
const adb = await this.getAdb();
|
|
623
|
+
const keyboardDismissStrategy = (null == options ? void 0 : options.keyboardDismissStrategy) ?? (null == (_this_options = this.options) ? void 0 : _this_options.keyboardDismissStrategy) ?? 'esc-first';
|
|
624
|
+
const keyboardStatus = await adb.isSoftKeyboardPresent();
|
|
625
|
+
const isKeyboardShown = 'boolean' == typeof keyboardStatus ? keyboardStatus : null == keyboardStatus ? void 0 : keyboardStatus.isKeyboardShown;
|
|
626
|
+
if (!isKeyboardShown) {
|
|
627
|
+
debugPage('Keyboard has no UI; no closing necessary');
|
|
628
|
+
return false;
|
|
629
|
+
}
|
|
630
|
+
const keyCodes = 'back-first' === keyboardDismissStrategy ? [
|
|
631
|
+
4,
|
|
632
|
+
111
|
|
633
|
+
] : [
|
|
634
|
+
111,
|
|
635
|
+
4
|
|
636
|
+
];
|
|
637
|
+
for (const keyCode of keyCodes){
|
|
638
|
+
await adb.keyevent(keyCode);
|
|
639
|
+
const startTime = Date.now();
|
|
640
|
+
const intervalMs = 100;
|
|
641
|
+
while(Date.now() - startTime < timeoutMs){
|
|
642
|
+
await sleep(intervalMs);
|
|
643
|
+
const currentStatus = await adb.isSoftKeyboardPresent();
|
|
644
|
+
const isStillShown = 'boolean' == typeof currentStatus ? currentStatus : null == currentStatus ? void 0 : currentStatus.isKeyboardShown;
|
|
645
|
+
if (!isStillShown) {
|
|
646
|
+
debugPage(`Keyboard hidden successfully with keycode ${keyCode}`);
|
|
647
|
+
return true;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
debugPage(`Keyboard still shown after keycode ${keyCode}, trying next key`);
|
|
651
|
+
}
|
|
652
|
+
console.warn('Warning: Failed to hide the software keyboard after trying both ESC and BACK keys');
|
|
653
|
+
return false;
|
|
654
|
+
}
|
|
655
|
+
constructor(deviceId, options){
|
|
656
|
+
_define_property(this, "deviceId", void 0);
|
|
657
|
+
_define_property(this, "yadbPushed", false);
|
|
658
|
+
_define_property(this, "devicePixelRatio", 1);
|
|
659
|
+
_define_property(this, "adb", null);
|
|
660
|
+
_define_property(this, "connectingAdb", null);
|
|
661
|
+
_define_property(this, "destroyed", false);
|
|
662
|
+
_define_property(this, "pageType", 'android');
|
|
663
|
+
_define_property(this, "uri", void 0);
|
|
664
|
+
_define_property(this, "options", void 0);
|
|
665
|
+
node_assert(deviceId, 'deviceId is required for AndroidDevice');
|
|
666
|
+
this.deviceId = deviceId;
|
|
667
|
+
this.options = options;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
async function getConnectedDevices() {
|
|
671
|
+
try {
|
|
672
|
+
const adb = await ADB.createADB({
|
|
673
|
+
adbExecTimeout: 60000
|
|
674
|
+
});
|
|
675
|
+
const devices = await adb.getConnectedDevices();
|
|
676
|
+
debugPage(`Found ${devices.length} connected devices: `, devices);
|
|
677
|
+
return devices;
|
|
678
|
+
} catch (error) {
|
|
679
|
+
console.error('Failed to get device list:', error);
|
|
680
|
+
throw new Error(`Unable to get connected Android device list, please check https://midscenejs.com/integrate-with-android.html#faq : ${error.message}`, {
|
|
681
|
+
cause: error
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
class AndroidAgent extends PageAgent {
|
|
686
|
+
async launch(uri) {
|
|
687
|
+
const device = this.page;
|
|
688
|
+
await device.launch(uri);
|
|
689
|
+
}
|
|
690
|
+
constructor(page, opts){
|
|
691
|
+
super(page, opts);
|
|
692
|
+
if (!vlLocateMode()) throw new Error('Android Agent only supports vl-model. https://midscenejs.com/choose-a-model.html');
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
async function agentFromAdbDevice(deviceId, opts) {
|
|
696
|
+
if (!deviceId) {
|
|
697
|
+
const devices = await getConnectedDevices();
|
|
698
|
+
deviceId = devices[0].udid;
|
|
699
|
+
debugPage('deviceId not specified, will use the first device (id = %s)', deviceId);
|
|
700
|
+
}
|
|
701
|
+
const page = new AndroidDevice(deviceId, {
|
|
702
|
+
autoDismissKeyboard: null == opts ? void 0 : opts.autoDismissKeyboard,
|
|
703
|
+
androidAdbPath: null == opts ? void 0 : opts.androidAdbPath,
|
|
704
|
+
remoteAdbHost: null == opts ? void 0 : opts.remoteAdbHost,
|
|
705
|
+
remoteAdbPort: null == opts ? void 0 : opts.remoteAdbPort,
|
|
706
|
+
imeStrategy: null == opts ? void 0 : opts.imeStrategy
|
|
707
|
+
});
|
|
708
|
+
await page.connect();
|
|
709
|
+
return new AndroidAgent(page, opts);
|
|
710
|
+
}
|
|
711
|
+
export { AndroidAgent, AndroidDevice, agentFromAdbDevice, getConnectedDevices, overrideAIConfig };
|