@huanlin/dsh-plugin-android-use 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +144 -0
- package/cordis.patch.yml +10 -0
- package/lib/client.js +326 -0
- package/lib/client.js.map +1 -0
- package/lib/index.d.ts +24 -0
- package/lib/index.js +2178 -0
- package/package.json +112 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,2178 @@
|
|
|
1
|
+
import z from "schemastery";
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
4
|
+
//#region src/adb.ts
|
|
5
|
+
/**
|
|
6
|
+
* adb.ts — spawn-based adb client with signal cancellation and binary support.
|
|
7
|
+
*
|
|
8
|
+
* Zero runtime dependencies: only `node:child_process` spawn. The class is
|
|
9
|
+
* injected into `registerTools` so tests substitute a fake without touching
|
|
10
|
+
* the real adb binary.
|
|
11
|
+
*
|
|
12
|
+
* Conventions (per plugin-development-guide.md §3):
|
|
13
|
+
* C5 — adb missing / device offline / spawn failure are infrastructure
|
|
14
|
+
* failures: they throw (the model sees a tool error, not a silent
|
|
15
|
+
* canonical value).
|
|
16
|
+
* C6 — every spawn receives `exec.signal`; aborting kills the child.
|
|
17
|
+
*
|
|
18
|
+
* @module @huanlin/dsh-plugin-android-use/src/adb
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Parse `adb devices -l` stdout into device rows.
|
|
22
|
+
*
|
|
23
|
+
* Example input:
|
|
24
|
+
* ```
|
|
25
|
+
* List of devices attached
|
|
26
|
+
* 192.168.5.15:43709 device product:PJF110 model:PJF110 device:OP5CFBL1 transport_id:1
|
|
27
|
+
* ```
|
|
28
|
+
* @param output - raw stdout from `adb devices -l`.
|
|
29
|
+
* @returns parsed device rows (empty when no devices are attached).
|
|
30
|
+
*/
|
|
31
|
+
function parseDeviceList(output) {
|
|
32
|
+
const lines = output.split(/\r?\n/);
|
|
33
|
+
const devices = [];
|
|
34
|
+
let started = false;
|
|
35
|
+
for (const line of lines) {
|
|
36
|
+
const trimmed = line.trim();
|
|
37
|
+
if (trimmed === "") continue;
|
|
38
|
+
if (trimmed === "List of devices attached") {
|
|
39
|
+
started = true;
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (!started) continue;
|
|
43
|
+
if (trimmed.startsWith("*")) continue;
|
|
44
|
+
const parts = trimmed.split(/\s+/);
|
|
45
|
+
if (parts.length < 2) continue;
|
|
46
|
+
const serial = parts[0];
|
|
47
|
+
const state = parts[1];
|
|
48
|
+
if (state === "device" || state === "offline" || state === "unauthorized" || state === "connecting") {
|
|
49
|
+
const device = {
|
|
50
|
+
serial,
|
|
51
|
+
state
|
|
52
|
+
};
|
|
53
|
+
for (let i = 2; i < parts.length; i++) {
|
|
54
|
+
const kv = parts[i];
|
|
55
|
+
const colon = kv.indexOf(":");
|
|
56
|
+
if (colon < 0) continue;
|
|
57
|
+
const key = kv.slice(0, colon);
|
|
58
|
+
const value = kv.slice(colon + 1);
|
|
59
|
+
if (key === "product") device.product = value;
|
|
60
|
+
else if (key === "model") device.model = value;
|
|
61
|
+
else if (key === "device") device.device = value;
|
|
62
|
+
else if (key === "transport_id") {
|
|
63
|
+
const id = Number(value);
|
|
64
|
+
if (Number.isInteger(id)) device.transportId = id;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
devices.push(device);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return devices;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Spawn-based adb client. All methods honor `signal` for cancellation (C6);
|
|
74
|
+
* a non-zero exit with empty stdout throws (infrastructure failure, C5).
|
|
75
|
+
*/
|
|
76
|
+
var AdbClient = class {
|
|
77
|
+
adbPath;
|
|
78
|
+
/**
|
|
79
|
+
* @param adbPath - path to the adb binary (default `'adb'`).
|
|
80
|
+
*/
|
|
81
|
+
constructor(adbPath = "adb") {
|
|
82
|
+
this.adbPath = adbPath;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Run adb collecting stdout/stderr as UTF-8 strings.
|
|
86
|
+
* @param args - full argument list (e.g. `['-s', serial, 'shell', 'wm size']`).
|
|
87
|
+
* @param options - serial and cancellation signal.
|
|
88
|
+
* @returns the collected stdout, stderr, and exit code.
|
|
89
|
+
*/
|
|
90
|
+
run(args, options = {}) {
|
|
91
|
+
return new Promise((resolve, reject) => {
|
|
92
|
+
const child = spawn(this.adbPath, args, {
|
|
93
|
+
stdio: [
|
|
94
|
+
"ignore",
|
|
95
|
+
"pipe",
|
|
96
|
+
"pipe"
|
|
97
|
+
],
|
|
98
|
+
windowsHide: true
|
|
99
|
+
});
|
|
100
|
+
const stdoutChunks = [];
|
|
101
|
+
let stderrChunks = [];
|
|
102
|
+
child.stdout.on("data", (chunk) => {
|
|
103
|
+
stdoutChunks.push(chunk);
|
|
104
|
+
});
|
|
105
|
+
child.stderr.on("data", (chunk) => {
|
|
106
|
+
stderrChunks.push(chunk);
|
|
107
|
+
});
|
|
108
|
+
const onAbort = () => {
|
|
109
|
+
if (!child.killed) child.kill("SIGTERM");
|
|
110
|
+
};
|
|
111
|
+
const signal = options.signal;
|
|
112
|
+
if (signal !== void 0) {
|
|
113
|
+
if (signal.aborted) child.kill("SIGTERM");
|
|
114
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
115
|
+
}
|
|
116
|
+
child.on("error", (error) => {
|
|
117
|
+
if (signal !== void 0) signal.removeEventListener("abort", onAbort);
|
|
118
|
+
reject(/* @__PURE__ */ new Error(`adb spawn failed: ${error.message}`));
|
|
119
|
+
});
|
|
120
|
+
child.on("close", (code) => {
|
|
121
|
+
if (signal !== void 0) signal.removeEventListener("abort", onAbort);
|
|
122
|
+
resolve({
|
|
123
|
+
stdout: Buffer.concat(stdoutChunks).toString("utf8"),
|
|
124
|
+
stderr: Buffer.concat(stderrChunks).toString("utf8"),
|
|
125
|
+
exitCode: code ?? -1
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Run adb collecting stdout as a raw Buffer (binary-safe, for `screencap`).
|
|
132
|
+
* @param args - full argument list.
|
|
133
|
+
* @param options - serial and cancellation signal.
|
|
134
|
+
* @returns the raw stdout buffer.
|
|
135
|
+
*/
|
|
136
|
+
runBinary(args, options = {}) {
|
|
137
|
+
return new Promise((resolve, reject) => {
|
|
138
|
+
const child = spawn(this.adbPath, args, {
|
|
139
|
+
stdio: [
|
|
140
|
+
"ignore",
|
|
141
|
+
"pipe",
|
|
142
|
+
"pipe"
|
|
143
|
+
],
|
|
144
|
+
windowsHide: true
|
|
145
|
+
});
|
|
146
|
+
const stdoutChunks = [];
|
|
147
|
+
const stderrChunks = [];
|
|
148
|
+
child.stdout.on("data", (chunk) => {
|
|
149
|
+
stdoutChunks.push(chunk);
|
|
150
|
+
});
|
|
151
|
+
child.stderr.on("data", (chunk) => {
|
|
152
|
+
stderrChunks.push(chunk);
|
|
153
|
+
});
|
|
154
|
+
const onAbort = () => {
|
|
155
|
+
if (!child.killed) child.kill("SIGTERM");
|
|
156
|
+
};
|
|
157
|
+
const signal = options.signal;
|
|
158
|
+
if (signal !== void 0) {
|
|
159
|
+
if (signal.aborted) child.kill("SIGTERM");
|
|
160
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
161
|
+
}
|
|
162
|
+
child.on("error", (error) => {
|
|
163
|
+
if (signal !== void 0) signal.removeEventListener("abort", onAbort);
|
|
164
|
+
reject(/* @__PURE__ */ new Error(`adb spawn failed: ${error.message}`));
|
|
165
|
+
});
|
|
166
|
+
child.on("close", (code) => {
|
|
167
|
+
if (signal !== void 0) signal.removeEventListener("abort", onAbort);
|
|
168
|
+
const stdout = Buffer.concat(stdoutChunks);
|
|
169
|
+
const stderr = Buffer.concat(stderrChunks).toString("utf8");
|
|
170
|
+
if (code !== 0) {
|
|
171
|
+
reject(/* @__PURE__ */ new Error(`adb exited with code ${code}: ${stderr || "(no stderr)"}`));
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
if (signal?.aborted) {
|
|
175
|
+
reject(/* @__PURE__ */ new Error("adb was cancelled"));
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
resolve(stdout);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* List attached devices via `adb devices -l`.
|
|
184
|
+
* @returns parsed device rows.
|
|
185
|
+
*/
|
|
186
|
+
async devices(signal) {
|
|
187
|
+
return parseDeviceList((await this.run(["devices", "-l"], { signal })).stdout);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Run a shell command on the device (`adb -s <serial> shell <cmd>`).
|
|
191
|
+
* @returns the command stdout as a string.
|
|
192
|
+
*/
|
|
193
|
+
async shell(serial, command, signal) {
|
|
194
|
+
const result = await this.run([
|
|
195
|
+
"-s",
|
|
196
|
+
serial,
|
|
197
|
+
"shell",
|
|
198
|
+
command
|
|
199
|
+
], {
|
|
200
|
+
serial,
|
|
201
|
+
signal
|
|
202
|
+
});
|
|
203
|
+
if (result.exitCode !== 0) throw new Error(`adb shell failed (exit ${result.exitCode}): ${result.stderr || result.stdout}`);
|
|
204
|
+
return result.stdout;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Run an exec-out command (`adb -s <serial> exec-out <cmd>`), binary-safe.
|
|
208
|
+
* @returns the command stdout as a string (UTF-8 decoded).
|
|
209
|
+
*/
|
|
210
|
+
async execOut(serial, command, signal) {
|
|
211
|
+
const result = await this.run([
|
|
212
|
+
"-s",
|
|
213
|
+
serial,
|
|
214
|
+
"exec-out",
|
|
215
|
+
command
|
|
216
|
+
], {
|
|
217
|
+
serial,
|
|
218
|
+
signal
|
|
219
|
+
});
|
|
220
|
+
if (result.exitCode !== 0) throw new Error(`adb exec-out failed (exit ${result.exitCode}): ${result.stderr}`);
|
|
221
|
+
return result.stdout;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Run an exec-out command collecting raw bytes (for `screencap -p`).
|
|
225
|
+
* @returns the command stdout as a Buffer.
|
|
226
|
+
*/
|
|
227
|
+
async execOutBinary(serial, command, signal) {
|
|
228
|
+
return this.runBinary([
|
|
229
|
+
"-s",
|
|
230
|
+
serial,
|
|
231
|
+
"exec-out",
|
|
232
|
+
command
|
|
233
|
+
], {
|
|
234
|
+
serial,
|
|
235
|
+
signal
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
/** Parse `wm size` output into `{ width, height }`. */
|
|
240
|
+
function parseWmSize$1(output) {
|
|
241
|
+
const overrideMatch = /Override size:\s*(\d+)x(\d+)/.exec(output);
|
|
242
|
+
if (overrideMatch !== null) return {
|
|
243
|
+
width: Number(overrideMatch[1]),
|
|
244
|
+
height: Number(overrideMatch[2])
|
|
245
|
+
};
|
|
246
|
+
const physicalMatch = /Physical size:\s*(\d+)x(\d+)/.exec(output);
|
|
247
|
+
if (physicalMatch !== null) return {
|
|
248
|
+
width: Number(physicalMatch[1]),
|
|
249
|
+
height: Number(physicalMatch[2])
|
|
250
|
+
};
|
|
251
|
+
throw new Error(`could not parse "wm size" output: ${output.trim()}`);
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Resolve the target device serial following the priority order:
|
|
255
|
+
* 1. `args.serial` (explicit per-call parameter)
|
|
256
|
+
* 2. `config.defaultSerial` (plugin config)
|
|
257
|
+
* 3. single attached device (auto-select)
|
|
258
|
+
* 4. multiple devices → throw "multiple devices, pass serial"
|
|
259
|
+
*
|
|
260
|
+
* @param adb - the adb client (used to list devices when needed).
|
|
261
|
+
* @param argSerial - the per-call `serial` argument, if the model supplied one.
|
|
262
|
+
* @param configSerial - the plugin config `defaultSerial`, if configured.
|
|
263
|
+
* @param signal - cancellation for the device-list query.
|
|
264
|
+
* @returns the resolved serial and its source.
|
|
265
|
+
*/
|
|
266
|
+
async function resolveSerial(adb, argSerial, configSerial, signal) {
|
|
267
|
+
if (argSerial !== void 0 && argSerial !== "") return {
|
|
268
|
+
serial: argSerial,
|
|
269
|
+
source: "param"
|
|
270
|
+
};
|
|
271
|
+
if (configSerial !== void 0 && configSerial !== "") return {
|
|
272
|
+
serial: configSerial,
|
|
273
|
+
source: "config"
|
|
274
|
+
};
|
|
275
|
+
const ready = (await adb.devices(signal)).filter((d) => d.state === "device");
|
|
276
|
+
if (ready.length === 0) throw new Error("no device is ready; connect a device or specify a serial");
|
|
277
|
+
if (ready.length > 1) {
|
|
278
|
+
const list = ready.map((d) => d.serial).join(", ");
|
|
279
|
+
throw new Error(`multiple devices attached (${list}); pass the "serial" parameter to select one`);
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
serial: ready[0].serial,
|
|
283
|
+
source: "auto"
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
//#endregion
|
|
287
|
+
//#region src/tools/device.ts
|
|
288
|
+
/** Parse `wm size` output, preferring the override (effective) resolution. */
|
|
289
|
+
function parseWmSize(output) {
|
|
290
|
+
const physical = /Physical size:\s*(\d+)x(\d+)/.exec(output);
|
|
291
|
+
const override = /Override size:\s*(\d+)x(\d+)/.exec(output);
|
|
292
|
+
const pw = physical !== null ? Number(physical[1]) : 0;
|
|
293
|
+
const ph = physical !== null ? Number(physical[2]) : 0;
|
|
294
|
+
return {
|
|
295
|
+
width: override !== null ? Number(override[1]) : pw,
|
|
296
|
+
height: override !== null ? Number(override[2]) : ph,
|
|
297
|
+
physicalWidth: pw,
|
|
298
|
+
physicalHeight: ph
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
/** Parse `wm density` output, preferring the override density. */
|
|
302
|
+
function parseWmDensity(output) {
|
|
303
|
+
const physical = /Physical density:\s*(\d+)/.exec(output);
|
|
304
|
+
const override = /Override density:\s*(\d+)/.exec(output);
|
|
305
|
+
const pd = physical !== null ? Number(physical[1]) : 0;
|
|
306
|
+
return {
|
|
307
|
+
density: override !== null ? Number(override[1]) : pd,
|
|
308
|
+
physicalDensity: pd
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
/** Parse `dumpsys power` output for the wakefulness state. */
|
|
312
|
+
function parseWakefulness$1(output) {
|
|
313
|
+
return /mWakefulness=Awake/.test(output);
|
|
314
|
+
}
|
|
315
|
+
/** Convert an `AdbDevice` to the canonical list-devices entry. */
|
|
316
|
+
function deviceToEntry(d) {
|
|
317
|
+
const entry = {
|
|
318
|
+
serial: d.serial,
|
|
319
|
+
state: d.state
|
|
320
|
+
};
|
|
321
|
+
if (d.product !== void 0) entry.product = d.product;
|
|
322
|
+
if (d.model !== void 0) entry.model = d.model;
|
|
323
|
+
if (d.device !== void 0) entry.device = d.device;
|
|
324
|
+
return entry;
|
|
325
|
+
}
|
|
326
|
+
function renderListDevices(value) {
|
|
327
|
+
if (value.devices.length === 0) return "No devices attached.";
|
|
328
|
+
const lines = [`Found ${value.devices.length} device(s):`];
|
|
329
|
+
for (const d of value.devices) {
|
|
330
|
+
const parts = [`state: ${d.state}`];
|
|
331
|
+
if (d.model !== void 0) parts.push(`model: ${d.model}`);
|
|
332
|
+
if (d.product !== void 0) parts.push(`product: ${d.product}`);
|
|
333
|
+
lines.push(` - ${d.serial} (${parts.join(", ")})`);
|
|
334
|
+
}
|
|
335
|
+
return lines.join("\n");
|
|
336
|
+
}
|
|
337
|
+
function renderDeviceInfo(value) {
|
|
338
|
+
return [
|
|
339
|
+
`Device: ${value.serial}`,
|
|
340
|
+
` Model: ${value.model} (${value.brand})`,
|
|
341
|
+
` Android: ${value.android_version} (SDK ${value.sdk})`,
|
|
342
|
+
` Screen: ${value.screen.width}x${value.screen.height} @ ${value.screen.density}dpi`,
|
|
343
|
+
` Screen on: ${value.screen_on}`
|
|
344
|
+
].join("\n");
|
|
345
|
+
}
|
|
346
|
+
function textRender$2(fn) {
|
|
347
|
+
return (_args, value) => [{
|
|
348
|
+
type: "text",
|
|
349
|
+
text: fn(value)
|
|
350
|
+
}];
|
|
351
|
+
}
|
|
352
|
+
/** Register `android_list_devices` and `android_device_info`. */
|
|
353
|
+
function registerDeviceTools(ctx, deps) {
|
|
354
|
+
ctx.tools.register(defineTool({
|
|
355
|
+
name: "android_list_devices",
|
|
356
|
+
description: "List all Android devices attached via adb. Returns each device's serial, state, product, model, and device name. Call this first to discover available devices before using other android_* tools.",
|
|
357
|
+
parameters: {},
|
|
358
|
+
output: {
|
|
359
|
+
schema: {
|
|
360
|
+
type: "object",
|
|
361
|
+
additionalProperties: false,
|
|
362
|
+
properties: {
|
|
363
|
+
devices: {
|
|
364
|
+
type: "array",
|
|
365
|
+
items: {
|
|
366
|
+
type: "object",
|
|
367
|
+
additionalProperties: false,
|
|
368
|
+
properties: {
|
|
369
|
+
serial: {
|
|
370
|
+
type: "string",
|
|
371
|
+
required: true
|
|
372
|
+
},
|
|
373
|
+
state: {
|
|
374
|
+
type: "string",
|
|
375
|
+
required: true
|
|
376
|
+
},
|
|
377
|
+
product: { type: "string" },
|
|
378
|
+
model: { type: "string" },
|
|
379
|
+
device: { type: "string" }
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
},
|
|
383
|
+
total: {
|
|
384
|
+
type: "integer",
|
|
385
|
+
required: true
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
},
|
|
389
|
+
render: textRender$2(renderListDevices)
|
|
390
|
+
},
|
|
391
|
+
async execute(_args, exec) {
|
|
392
|
+
const devices = await deps.adb.devices(exec.signal);
|
|
393
|
+
return {
|
|
394
|
+
devices: devices.map(deviceToEntry),
|
|
395
|
+
total: devices.length
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
}));
|
|
399
|
+
ctx.tools.register(defineTool({
|
|
400
|
+
name: "android_device_info",
|
|
401
|
+
description: "Get detailed information about an Android device: model, brand, Android version, SDK level, screen resolution, density, and whether the screen is currently on. Pass \"serial\" to target a specific device, or omit it when only one device is attached.",
|
|
402
|
+
parameters: { serial: {
|
|
403
|
+
type: "string",
|
|
404
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
405
|
+
} },
|
|
406
|
+
output: {
|
|
407
|
+
schema: {
|
|
408
|
+
type: "object",
|
|
409
|
+
additionalProperties: false,
|
|
410
|
+
properties: {
|
|
411
|
+
serial: {
|
|
412
|
+
type: "string",
|
|
413
|
+
required: true
|
|
414
|
+
},
|
|
415
|
+
model: {
|
|
416
|
+
type: "string",
|
|
417
|
+
required: true
|
|
418
|
+
},
|
|
419
|
+
brand: {
|
|
420
|
+
type: "string",
|
|
421
|
+
required: true
|
|
422
|
+
},
|
|
423
|
+
android_version: {
|
|
424
|
+
type: "string",
|
|
425
|
+
required: true
|
|
426
|
+
},
|
|
427
|
+
sdk: {
|
|
428
|
+
type: "integer",
|
|
429
|
+
required: true
|
|
430
|
+
},
|
|
431
|
+
screen: {
|
|
432
|
+
type: "object",
|
|
433
|
+
additionalProperties: false,
|
|
434
|
+
required: true,
|
|
435
|
+
properties: {
|
|
436
|
+
width: {
|
|
437
|
+
type: "integer",
|
|
438
|
+
required: true
|
|
439
|
+
},
|
|
440
|
+
height: {
|
|
441
|
+
type: "integer",
|
|
442
|
+
required: true
|
|
443
|
+
},
|
|
444
|
+
density: {
|
|
445
|
+
type: "integer",
|
|
446
|
+
required: true
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
},
|
|
450
|
+
screen_on: {
|
|
451
|
+
type: "boolean",
|
|
452
|
+
required: true
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
},
|
|
456
|
+
render: textRender$2(renderDeviceInfo)
|
|
457
|
+
},
|
|
458
|
+
async execute(args, exec) {
|
|
459
|
+
const a = args;
|
|
460
|
+
const config = deps.getConfig();
|
|
461
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
462
|
+
const sections = splitByMarkers(await deps.adb.shell(serial, "echo \"=SZ=\"; wm size; echo \"=DN=\"; wm density; echo \"=MD=\"; getprop ro.product.model; echo \"=BR=\"; getprop ro.product.brand; echo \"=VR=\"; getprop ro.build.version.release; echo \"=SDK=\"; getprop ro.build.version.sdk; echo \"=WK=\"; dumpsys power|grep mWakefulness", exec.signal));
|
|
463
|
+
const sz = parseWmSize(sections.sz);
|
|
464
|
+
const dn = parseWmDensity(sections.dn);
|
|
465
|
+
return {
|
|
466
|
+
serial,
|
|
467
|
+
model: sections.md.trim(),
|
|
468
|
+
brand: sections.br.trim(),
|
|
469
|
+
android_version: sections.vr.trim(),
|
|
470
|
+
sdk: Number(sections.sdk.trim()) || 0,
|
|
471
|
+
screen: {
|
|
472
|
+
width: sz.width,
|
|
473
|
+
height: sz.height,
|
|
474
|
+
density: dn.density
|
|
475
|
+
},
|
|
476
|
+
screen_on: parseWakefulness$1(sections.wk)
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
}));
|
|
480
|
+
}
|
|
481
|
+
/** Split a batched shell output by `=XX=` markers into named sections. */
|
|
482
|
+
function splitByMarkers(output) {
|
|
483
|
+
const parts = output.split(/=[A-Z]+=/);
|
|
484
|
+
return {
|
|
485
|
+
sz: parts[1] ?? "",
|
|
486
|
+
dn: parts[2] ?? "",
|
|
487
|
+
md: parts[3] ?? "",
|
|
488
|
+
br: parts[4] ?? "",
|
|
489
|
+
vr: parts[5] ?? "",
|
|
490
|
+
sdk: parts[6] ?? "",
|
|
491
|
+
wk: parts[7] ?? ""
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
//#endregion
|
|
495
|
+
//#region src/route.ts
|
|
496
|
+
/**
|
|
497
|
+
* Best-effort check whether the current model route accepts image input.
|
|
498
|
+
* Returns false when the llm service is absent, the route cannot be resolved,
|
|
499
|
+
* or the resolved model does not declare image input. Never throws.
|
|
500
|
+
*/
|
|
501
|
+
async function routeIsImageCapable(ctx, exec, signal) {
|
|
502
|
+
const llm = ctx.get("llm");
|
|
503
|
+
if (llm === void 0) return false;
|
|
504
|
+
const agent = exec.agent;
|
|
505
|
+
if (agent === void 0) return false;
|
|
506
|
+
const routed = agent.session.requestHeader()?.config;
|
|
507
|
+
const provider = routed?.provider ?? agent.options.provider;
|
|
508
|
+
const model = routed?.model ?? agent.options.model;
|
|
509
|
+
if (provider === void 0 || model === void 0) return false;
|
|
510
|
+
try {
|
|
511
|
+
return (await llm.resolveModelInfo(provider, model, signal)).inputModalities?.includes("image") === true;
|
|
512
|
+
} catch {
|
|
513
|
+
return false;
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
//#endregion
|
|
517
|
+
//#region src/xml.ts
|
|
518
|
+
/**
|
|
519
|
+
* Parse a `[left,top][right,bottom]` bounds string.
|
|
520
|
+
* @param raw - the attribute value, e.g. `"[0,0][1080,2414]"`.
|
|
521
|
+
* @returns the numeric bounds, or null when the string does not match.
|
|
522
|
+
*/
|
|
523
|
+
function parseBounds(raw) {
|
|
524
|
+
const match = /^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/.exec(raw);
|
|
525
|
+
if (match === null) return null;
|
|
526
|
+
return {
|
|
527
|
+
left: Number(match[1]),
|
|
528
|
+
top: Number(match[2]),
|
|
529
|
+
right: Number(match[3]),
|
|
530
|
+
bottom: Number(match[4])
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
/** Compute the center of a bounds rect. */
|
|
534
|
+
function centerOfBounds(bounds) {
|
|
535
|
+
return {
|
|
536
|
+
x: Math.floor((bounds.left + bounds.right) / 2),
|
|
537
|
+
y: Math.floor((bounds.top + bounds.bottom) / 2)
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
/** Decode the five XML entity references uiautomator emits. */
|
|
541
|
+
function decodeEntities(value) {
|
|
542
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, "\"").replace(/'/g, "'");
|
|
543
|
+
}
|
|
544
|
+
/**
|
|
545
|
+
* Extract one attribute value from a `<node ...>` tag string.
|
|
546
|
+
* @param tag - the raw `<node ...` text up to the closing `>` or `/>`.
|
|
547
|
+
* @param name - the attribute name without quotes.
|
|
548
|
+
* @returns the decoded value, or empty string when absent.
|
|
549
|
+
*/
|
|
550
|
+
function attrValue(tag, name) {
|
|
551
|
+
const match = new RegExp(`${name}="((?:[^"&]|&(?:amp|lt|gt|quot|apos);)*)"`, "u").exec(tag);
|
|
552
|
+
if (match === null) return "";
|
|
553
|
+
return decodeEntities(match[1]);
|
|
554
|
+
}
|
|
555
|
+
/** Parse a boolean attribute: `"true"` → true, everything else → false. */
|
|
556
|
+
function boolAttr(tag, name) {
|
|
557
|
+
return attrValue(tag, name) === "true";
|
|
558
|
+
}
|
|
559
|
+
/**
|
|
560
|
+
* Scan the XML for all `<node>` opening, self-closing, and closing tags in
|
|
561
|
+
* document order. Closing tags are tracked so depth stays accurate.
|
|
562
|
+
*/
|
|
563
|
+
const TAG_PATTERN = /<(\/?)node\b([^>]*?)(\/?)>/gu;
|
|
564
|
+
function scanTags(xml) {
|
|
565
|
+
const tags = [];
|
|
566
|
+
let match;
|
|
567
|
+
const pattern = new RegExp(TAG_PATTERN);
|
|
568
|
+
while ((match = pattern.exec(xml)) !== null) {
|
|
569
|
+
const closing = match[1] === "/";
|
|
570
|
+
const selfClosing = !closing && match[3] === "/";
|
|
571
|
+
tags.push({
|
|
572
|
+
raw: match[0],
|
|
573
|
+
closing,
|
|
574
|
+
selfClosing
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
return tags;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Parse a uiautomator dump XML into a flat node list with screen metadata.
|
|
581
|
+
*
|
|
582
|
+
* The root `<hierarchy rotation="N">` element carries the rotation; the
|
|
583
|
+
* screen dimensions are read from the first (root) node's bounds, which
|
|
584
|
+
* uiautomator always emits as the full display rectangle.
|
|
585
|
+
*
|
|
586
|
+
* Closing `</node>` tags are tracked so `depth` accurately reflects the tree
|
|
587
|
+
* nesting level (root node = depth 0, its children = depth 1, etc.).
|
|
588
|
+
*
|
|
589
|
+
* @param xml - the raw XML string from `uiautomator dump` + `cat`.
|
|
590
|
+
* @returns the parsed dump. Empty/malformed XML yields zero nodes and
|
|
591
|
+
* zero dimensions rather than throwing (the model still gets a result).
|
|
592
|
+
*/
|
|
593
|
+
function parseUiDumpXml(xml) {
|
|
594
|
+
const rotationMatch = /rotation="(-?\d+)"/.exec(xml);
|
|
595
|
+
const rotation = rotationMatch !== null ? Number(rotationMatch[1]) : 0;
|
|
596
|
+
const tags = scanTags(xml);
|
|
597
|
+
const nodes = [];
|
|
598
|
+
let depth = 0;
|
|
599
|
+
let screenWidth = 0;
|
|
600
|
+
let screenHeight = 0;
|
|
601
|
+
let isFirstNode = true;
|
|
602
|
+
for (const tag of tags) {
|
|
603
|
+
if (tag.closing) {
|
|
604
|
+
if (depth > 0) depth -= 1;
|
|
605
|
+
continue;
|
|
606
|
+
}
|
|
607
|
+
const raw = tag.raw;
|
|
608
|
+
const boundsRaw = attrValue(raw, "bounds");
|
|
609
|
+
const bounds = boundsRaw !== "" ? parseBounds(boundsRaw) : null;
|
|
610
|
+
if (isFirstNode && bounds !== null) {
|
|
611
|
+
screenWidth = bounds.right;
|
|
612
|
+
screenHeight = bounds.bottom;
|
|
613
|
+
isFirstNode = false;
|
|
614
|
+
}
|
|
615
|
+
const node = {
|
|
616
|
+
text: attrValue(raw, "text"),
|
|
617
|
+
content_desc: attrValue(raw, "content-desc"),
|
|
618
|
+
resource_id: attrValue(raw, "resource-id"),
|
|
619
|
+
class: attrValue(raw, "class"),
|
|
620
|
+
package: attrValue(raw, "package"),
|
|
621
|
+
bounds,
|
|
622
|
+
center: bounds !== null ? centerOfBounds(bounds) : null,
|
|
623
|
+
clickable: boolAttr(raw, "clickable"),
|
|
624
|
+
long_clickable: boolAttr(raw, "long-clickable"),
|
|
625
|
+
focusable: boolAttr(raw, "focusable"),
|
|
626
|
+
scrollable: boolAttr(raw, "scrollable"),
|
|
627
|
+
enabled: boolAttr(raw, "enabled"),
|
|
628
|
+
password: boolAttr(raw, "password"),
|
|
629
|
+
selected: boolAttr(raw, "selected"),
|
|
630
|
+
checked: boolAttr(raw, "checked"),
|
|
631
|
+
depth
|
|
632
|
+
};
|
|
633
|
+
nodes.push(node);
|
|
634
|
+
if (!tag.selfClosing) depth += 1;
|
|
635
|
+
}
|
|
636
|
+
return {
|
|
637
|
+
rotation,
|
|
638
|
+
screen_width: screenWidth,
|
|
639
|
+
screen_height: screenHeight,
|
|
640
|
+
nodes
|
|
641
|
+
};
|
|
642
|
+
}
|
|
643
|
+
//#endregion
|
|
644
|
+
//#region src/image.ts
|
|
645
|
+
/**
|
|
646
|
+
* image.ts — fit a PNG within the attachment store's per-side pixel limit.
|
|
647
|
+
*
|
|
648
|
+
* The attachment store enforces `maxImageDimension` (2000px by default).
|
|
649
|
+
* This module downscales only when necessary, returning a scale factor so
|
|
650
|
+
* callers can map device-native coordinates to image space for annotation.
|
|
651
|
+
*
|
|
652
|
+
* @module @huanlin/dsh-plugin-android-use/src/image
|
|
653
|
+
*/
|
|
654
|
+
/**
|
|
655
|
+
* Compute the uniform scale factor applied when a screenshot of the given
|
|
656
|
+
* device resolution is fit within `maxDimension`.
|
|
657
|
+
*
|
|
658
|
+
* The scale is uniform (same for X and Y) because `fitToMaxDimension` uses
|
|
659
|
+
* `fit: 'inside'` which preserves aspect ratio.
|
|
660
|
+
*
|
|
661
|
+
* @returns a value in (0, 1]. Returns 1 when no scaling is needed.
|
|
662
|
+
*/
|
|
663
|
+
function computeScale(deviceWidth, deviceHeight, maxDimension) {
|
|
664
|
+
if (deviceWidth <= 0 || deviceHeight <= 0) return 1;
|
|
665
|
+
return Math.min(1, maxDimension / Math.max(deviceWidth, deviceHeight));
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Downscale a PNG so its longest side fits within `maxDimension`, preserving
|
|
669
|
+
* aspect ratio. Returns the original bytes unchanged when already within the
|
|
670
|
+
* limit. Uses `sharp` for the resize.
|
|
671
|
+
*
|
|
672
|
+
* @param data - the raw PNG bytes from `screencap -p`.
|
|
673
|
+
* @param maxDimension - the attachment store's per-side pixel limit.
|
|
674
|
+
*/
|
|
675
|
+
async function fitToMaxDimension(data, maxDimension) {
|
|
676
|
+
const sharp = (await import("sharp")).default;
|
|
677
|
+
let meta;
|
|
678
|
+
try {
|
|
679
|
+
meta = await sharp(data).metadata();
|
|
680
|
+
} catch {
|
|
681
|
+
throw new Error("screencap returned an image with no dimensions");
|
|
682
|
+
}
|
|
683
|
+
const origW = meta.width ?? 0;
|
|
684
|
+
const origH = meta.height ?? 0;
|
|
685
|
+
if (origW === 0 || origH === 0) throw new Error("screencap returned an image with no dimensions");
|
|
686
|
+
if (Math.max(origW, origH) <= maxDimension) return {
|
|
687
|
+
data,
|
|
688
|
+
width: origW,
|
|
689
|
+
height: origH,
|
|
690
|
+
scaleX: 1,
|
|
691
|
+
scaleY: 1
|
|
692
|
+
};
|
|
693
|
+
const result = await sharp(data, {
|
|
694
|
+
failOn: "error",
|
|
695
|
+
limitInputPixels: false
|
|
696
|
+
}).resize({
|
|
697
|
+
width: maxDimension,
|
|
698
|
+
height: maxDimension,
|
|
699
|
+
fit: "inside",
|
|
700
|
+
withoutEnlargement: true
|
|
701
|
+
}).png().toBuffer({ resolveWithObject: true });
|
|
702
|
+
return {
|
|
703
|
+
data: new Uint8Array(result.data),
|
|
704
|
+
width: result.info.width,
|
|
705
|
+
height: result.info.height,
|
|
706
|
+
scaleX: result.info.width / origW,
|
|
707
|
+
scaleY: result.info.height / origH
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
//#endregion
|
|
711
|
+
//#region src/tools/screen.ts
|
|
712
|
+
function renderScreenshot(_args, value) {
|
|
713
|
+
const v = value;
|
|
714
|
+
const text = [
|
|
715
|
+
`Screenshot captured: ${v.width}x${v.height} px, ${v.bytes} bytes (image/png).`,
|
|
716
|
+
`Device resolution: ${v.device_width}x${v.device_height}, scale: ${v.scale.toFixed(4)}.`,
|
|
717
|
+
`Coordinates from android_ui_dump and android_tap use the ${v.width}x${v.height} image space.`,
|
|
718
|
+
`Image emitted to model context: ${v.image_emitted ? "yes" : "no"}${v.image_emitted ? "" : " (current route is not image-capable; use android_ui_dump for screen perception)"}.`
|
|
719
|
+
].join("\n");
|
|
720
|
+
if (v.image_emitted) {
|
|
721
|
+
const ref = {
|
|
722
|
+
attachmentId: v.image.attachmentId,
|
|
723
|
+
mediaType: "image/png",
|
|
724
|
+
bytes: v.image.bytes,
|
|
725
|
+
width: v.image.width,
|
|
726
|
+
height: v.image.height
|
|
727
|
+
};
|
|
728
|
+
return [{
|
|
729
|
+
type: "text",
|
|
730
|
+
text
|
|
731
|
+
}, {
|
|
732
|
+
type: "image",
|
|
733
|
+
attachment: ref
|
|
734
|
+
}];
|
|
735
|
+
}
|
|
736
|
+
return [{
|
|
737
|
+
type: "text",
|
|
738
|
+
text
|
|
739
|
+
}];
|
|
740
|
+
}
|
|
741
|
+
function nodeToTextLine(node, index) {
|
|
742
|
+
const parts = [`[${index}]`];
|
|
743
|
+
if (node.text !== "") parts.push(`text=${JSON.stringify(node.text)}`);
|
|
744
|
+
if (node.content_desc !== "") parts.push(`desc=${JSON.stringify(node.content_desc)}`);
|
|
745
|
+
if (node.resource_id !== "") parts.push(`id=${node.resource_id}`);
|
|
746
|
+
parts.push(`class=${node.class}`);
|
|
747
|
+
if (node.bounds !== null) parts.push(`bounds=[${node.bounds.left},${node.bounds.top}][${node.bounds.right},${node.bounds.bottom}]`);
|
|
748
|
+
if (node.center !== null) parts.push(`center=(${node.center.x},${node.center.y})`);
|
|
749
|
+
const flags = [];
|
|
750
|
+
if (node.clickable) flags.push("clickable");
|
|
751
|
+
if (node.long_clickable) flags.push("long-clickable");
|
|
752
|
+
if (node.scrollable) flags.push("scrollable");
|
|
753
|
+
if (node.focusable) flags.push("focusable");
|
|
754
|
+
if (!node.enabled) flags.push("disabled");
|
|
755
|
+
if (node.checked) flags.push("checked");
|
|
756
|
+
if (node.password) flags.push("password");
|
|
757
|
+
if (node.selected) flags.push("selected");
|
|
758
|
+
if (flags.length > 0) parts.push(flags.join(","));
|
|
759
|
+
return ` ${parts.join(" | ")}`;
|
|
760
|
+
}
|
|
761
|
+
function scaleDump(dump, scale) {
|
|
762
|
+
if (scale >= 1) return dump;
|
|
763
|
+
const r = (n) => Math.round(n * scale);
|
|
764
|
+
const nodes = dump.nodes.map((node) => ({
|
|
765
|
+
...node,
|
|
766
|
+
bounds: node.bounds !== null ? {
|
|
767
|
+
left: r(node.bounds.left),
|
|
768
|
+
top: r(node.bounds.top),
|
|
769
|
+
right: r(node.bounds.right),
|
|
770
|
+
bottom: r(node.bounds.bottom)
|
|
771
|
+
} : null,
|
|
772
|
+
center: node.center !== null ? {
|
|
773
|
+
x: r(node.center.x),
|
|
774
|
+
y: r(node.center.y)
|
|
775
|
+
} : null
|
|
776
|
+
}));
|
|
777
|
+
return {
|
|
778
|
+
rotation: dump.rotation,
|
|
779
|
+
screen_width: dump.screen_width,
|
|
780
|
+
screen_height: dump.screen_height,
|
|
781
|
+
nodes
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
function renderUiDump(_args, value) {
|
|
785
|
+
const v = value;
|
|
786
|
+
const total = v.nodes.length;
|
|
787
|
+
const interactive = v.nodes.filter((n) => n.text !== "" || n.content_desc !== "" || n.clickable || n.long_clickable || n.scrollable || n.focusable);
|
|
788
|
+
const lines = [
|
|
789
|
+
`UI dump: ${total} nodes total (${interactive.length} interactive/text), screen ${v.screen_width}x${v.screen_height}, rotation ${v.rotation}.`,
|
|
790
|
+
`Coordinates are in screenshot image space (${v.image_width}x${v.image_height}, scale ${v.scale.toFixed(4)}). Use these center values directly with android_tap.`,
|
|
791
|
+
`Showing ${interactive.length} interactive/text nodes:`
|
|
792
|
+
];
|
|
793
|
+
let idx = 0;
|
|
794
|
+
for (const node of v.nodes) {
|
|
795
|
+
if (!(node.text !== "" || node.content_desc !== "" || node.clickable || node.long_clickable || node.scrollable || node.focusable)) continue;
|
|
796
|
+
lines.push(nodeToTextLine(node, idx));
|
|
797
|
+
idx++;
|
|
798
|
+
}
|
|
799
|
+
return [{
|
|
800
|
+
type: "text",
|
|
801
|
+
text: lines.join("\n")
|
|
802
|
+
}];
|
|
803
|
+
}
|
|
804
|
+
/** Register `android_screenshot` and `android_ui_dump`. */
|
|
805
|
+
function registerScreenTools(ctx, deps) {
|
|
806
|
+
ctx.tools.register(defineTool({
|
|
807
|
+
name: "android_screenshot",
|
|
808
|
+
description: "Capture a screenshot from the Android device. The image may be scaled to fit the attachment store pixel limit; the result includes device_width, device_height, and scale so you know the mapping. Coordinates from android_ui_dump and android_tap use the scaled image space. The image is always saved to the attachment store (visible in the UI card). The image is emitted to the model only when the current model route accepts image input; otherwise, use android_ui_dump for text-based screen perception. Pass \"serial\" to target a specific device.",
|
|
809
|
+
parameters: { serial: {
|
|
810
|
+
type: "string",
|
|
811
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
812
|
+
} },
|
|
813
|
+
output: {
|
|
814
|
+
schema: {
|
|
815
|
+
type: "object",
|
|
816
|
+
additionalProperties: false,
|
|
817
|
+
properties: {
|
|
818
|
+
serial: {
|
|
819
|
+
type: "string",
|
|
820
|
+
required: true
|
|
821
|
+
},
|
|
822
|
+
width: {
|
|
823
|
+
type: "integer",
|
|
824
|
+
required: true
|
|
825
|
+
},
|
|
826
|
+
height: {
|
|
827
|
+
type: "integer",
|
|
828
|
+
required: true
|
|
829
|
+
},
|
|
830
|
+
bytes: {
|
|
831
|
+
type: "integer",
|
|
832
|
+
required: true
|
|
833
|
+
},
|
|
834
|
+
device_width: {
|
|
835
|
+
type: "integer",
|
|
836
|
+
required: true
|
|
837
|
+
},
|
|
838
|
+
device_height: {
|
|
839
|
+
type: "integer",
|
|
840
|
+
required: true
|
|
841
|
+
},
|
|
842
|
+
scale: {
|
|
843
|
+
type: "number",
|
|
844
|
+
required: true
|
|
845
|
+
},
|
|
846
|
+
image: {
|
|
847
|
+
type: "object",
|
|
848
|
+
additionalProperties: false,
|
|
849
|
+
required: true,
|
|
850
|
+
properties: {
|
|
851
|
+
attachmentId: {
|
|
852
|
+
type: "string",
|
|
853
|
+
required: true
|
|
854
|
+
},
|
|
855
|
+
mediaType: {
|
|
856
|
+
type: "string",
|
|
857
|
+
const: "image/png",
|
|
858
|
+
required: true
|
|
859
|
+
},
|
|
860
|
+
bytes: {
|
|
861
|
+
type: "integer",
|
|
862
|
+
required: true
|
|
863
|
+
},
|
|
864
|
+
width: {
|
|
865
|
+
type: "integer",
|
|
866
|
+
required: true
|
|
867
|
+
},
|
|
868
|
+
height: {
|
|
869
|
+
type: "integer",
|
|
870
|
+
required: true
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
},
|
|
874
|
+
image_emitted: {
|
|
875
|
+
type: "boolean",
|
|
876
|
+
required: true
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
},
|
|
880
|
+
render: renderScreenshot
|
|
881
|
+
},
|
|
882
|
+
async execute(args, exec) {
|
|
883
|
+
const a = args;
|
|
884
|
+
const config = deps.getConfig();
|
|
885
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
886
|
+
const attachments = ctx.get("attachments");
|
|
887
|
+
if (attachments === void 0) throw new Error("cannot capture screenshot: no attachment service is mounted");
|
|
888
|
+
const raw = await deps.adb.execOutBinary(serial, "screencap -p", exec.signal);
|
|
889
|
+
const fitted = await fitToMaxDimension(new Uint8Array(raw), attachments.imageLimits.maxImageDimension);
|
|
890
|
+
const ref = await attachments.saveImage({
|
|
891
|
+
data: fitted.data,
|
|
892
|
+
mediaType: "image/png",
|
|
893
|
+
name: "screenshot.png"
|
|
894
|
+
});
|
|
895
|
+
const imageEmitted = await routeIsImageCapable(ctx, exec, exec.signal);
|
|
896
|
+
const scale = fitted.scaleX;
|
|
897
|
+
return {
|
|
898
|
+
serial,
|
|
899
|
+
width: ref.width,
|
|
900
|
+
height: ref.height,
|
|
901
|
+
bytes: ref.bytes,
|
|
902
|
+
device_width: Math.round(ref.width / fitted.scaleX),
|
|
903
|
+
device_height: Math.round(ref.height / fitted.scaleY),
|
|
904
|
+
scale,
|
|
905
|
+
image: {
|
|
906
|
+
attachmentId: ref.attachmentId,
|
|
907
|
+
mediaType: "image/png",
|
|
908
|
+
bytes: ref.bytes,
|
|
909
|
+
width: ref.width,
|
|
910
|
+
height: ref.height
|
|
911
|
+
},
|
|
912
|
+
image_emitted: imageEmitted
|
|
913
|
+
};
|
|
914
|
+
}
|
|
915
|
+
}));
|
|
916
|
+
ctx.tools.register(defineTool({
|
|
917
|
+
name: "android_ui_dump",
|
|
918
|
+
description: "Dump the Android accessibility tree (UI hierarchy) as a structured node list. Each node includes text, content description, resource-id, class, bounds, center coordinates, and interaction flags (clickable, scrollable, etc.). All coordinates are in screenshot image space (scaled to match the screenshot from android_screenshot). Use node \"center\" values directly with android_tap. Pass \"serial\" to target a specific device.",
|
|
919
|
+
parameters: { serial: {
|
|
920
|
+
type: "string",
|
|
921
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
922
|
+
} },
|
|
923
|
+
output: {
|
|
924
|
+
schema: {
|
|
925
|
+
type: "object",
|
|
926
|
+
additionalProperties: false,
|
|
927
|
+
properties: {
|
|
928
|
+
serial: {
|
|
929
|
+
type: "string",
|
|
930
|
+
required: true
|
|
931
|
+
},
|
|
932
|
+
screen_width: {
|
|
933
|
+
type: "integer",
|
|
934
|
+
required: true
|
|
935
|
+
},
|
|
936
|
+
screen_height: {
|
|
937
|
+
type: "integer",
|
|
938
|
+
required: true
|
|
939
|
+
},
|
|
940
|
+
image_width: {
|
|
941
|
+
type: "integer",
|
|
942
|
+
required: true
|
|
943
|
+
},
|
|
944
|
+
image_height: {
|
|
945
|
+
type: "integer",
|
|
946
|
+
required: true
|
|
947
|
+
},
|
|
948
|
+
scale: {
|
|
949
|
+
type: "number",
|
|
950
|
+
required: true
|
|
951
|
+
},
|
|
952
|
+
rotation: {
|
|
953
|
+
type: "integer",
|
|
954
|
+
required: true
|
|
955
|
+
},
|
|
956
|
+
nodes: {
|
|
957
|
+
type: "array",
|
|
958
|
+
items: {
|
|
959
|
+
type: "object",
|
|
960
|
+
additionalProperties: false,
|
|
961
|
+
properties: {
|
|
962
|
+
text: {
|
|
963
|
+
type: "string",
|
|
964
|
+
required: true
|
|
965
|
+
},
|
|
966
|
+
content_desc: {
|
|
967
|
+
type: "string",
|
|
968
|
+
required: true
|
|
969
|
+
},
|
|
970
|
+
resource_id: {
|
|
971
|
+
type: "string",
|
|
972
|
+
required: true
|
|
973
|
+
},
|
|
974
|
+
class: {
|
|
975
|
+
type: "string",
|
|
976
|
+
required: true
|
|
977
|
+
},
|
|
978
|
+
package: {
|
|
979
|
+
type: "string",
|
|
980
|
+
required: true
|
|
981
|
+
},
|
|
982
|
+
bounds: {
|
|
983
|
+
oneOf: [{ type: "null" }, {
|
|
984
|
+
type: "object",
|
|
985
|
+
additionalProperties: false,
|
|
986
|
+
properties: {
|
|
987
|
+
left: {
|
|
988
|
+
type: "integer",
|
|
989
|
+
required: true
|
|
990
|
+
},
|
|
991
|
+
top: {
|
|
992
|
+
type: "integer",
|
|
993
|
+
required: true
|
|
994
|
+
},
|
|
995
|
+
right: {
|
|
996
|
+
type: "integer",
|
|
997
|
+
required: true
|
|
998
|
+
},
|
|
999
|
+
bottom: {
|
|
1000
|
+
type: "integer",
|
|
1001
|
+
required: true
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
}],
|
|
1005
|
+
required: true
|
|
1006
|
+
},
|
|
1007
|
+
center: {
|
|
1008
|
+
oneOf: [{ type: "null" }, {
|
|
1009
|
+
type: "object",
|
|
1010
|
+
additionalProperties: false,
|
|
1011
|
+
properties: {
|
|
1012
|
+
x: {
|
|
1013
|
+
type: "integer",
|
|
1014
|
+
required: true
|
|
1015
|
+
},
|
|
1016
|
+
y: {
|
|
1017
|
+
type: "integer",
|
|
1018
|
+
required: true
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
}],
|
|
1022
|
+
required: true
|
|
1023
|
+
},
|
|
1024
|
+
clickable: {
|
|
1025
|
+
type: "boolean",
|
|
1026
|
+
required: true
|
|
1027
|
+
},
|
|
1028
|
+
long_clickable: {
|
|
1029
|
+
type: "boolean",
|
|
1030
|
+
required: true
|
|
1031
|
+
},
|
|
1032
|
+
focusable: {
|
|
1033
|
+
type: "boolean",
|
|
1034
|
+
required: true
|
|
1035
|
+
},
|
|
1036
|
+
scrollable: {
|
|
1037
|
+
type: "boolean",
|
|
1038
|
+
required: true
|
|
1039
|
+
},
|
|
1040
|
+
enabled: {
|
|
1041
|
+
type: "boolean",
|
|
1042
|
+
required: true
|
|
1043
|
+
},
|
|
1044
|
+
password: {
|
|
1045
|
+
type: "boolean",
|
|
1046
|
+
required: true
|
|
1047
|
+
},
|
|
1048
|
+
selected: {
|
|
1049
|
+
type: "boolean",
|
|
1050
|
+
required: true
|
|
1051
|
+
},
|
|
1052
|
+
checked: {
|
|
1053
|
+
type: "boolean",
|
|
1054
|
+
required: true
|
|
1055
|
+
},
|
|
1056
|
+
depth: {
|
|
1057
|
+
type: "integer",
|
|
1058
|
+
required: true
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
},
|
|
1065
|
+
render: renderUiDump
|
|
1066
|
+
},
|
|
1067
|
+
async execute(args, exec) {
|
|
1068
|
+
const a = args;
|
|
1069
|
+
const config = deps.getConfig();
|
|
1070
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
1071
|
+
const dumpPath = `/sdcard/dsh_ui_dump_${Date.now()}.xml`;
|
|
1072
|
+
await deps.adb.shell(serial, `uiautomator dump ${dumpPath}`, exec.signal);
|
|
1073
|
+
const dump = parseUiDumpXml(await deps.adb.execOut(serial, `cat ${dumpPath}`, exec.signal));
|
|
1074
|
+
const maxDim = ctx.get("attachments")?.imageLimits.maxImageDimension ?? Infinity;
|
|
1075
|
+
const scale = computeScale(dump.screen_width, dump.screen_height, maxDim);
|
|
1076
|
+
const scaled = scaleDump(dump, scale);
|
|
1077
|
+
return {
|
|
1078
|
+
serial,
|
|
1079
|
+
screen_width: dump.screen_width,
|
|
1080
|
+
screen_height: dump.screen_height,
|
|
1081
|
+
image_width: Math.round(dump.screen_width * scale),
|
|
1082
|
+
image_height: Math.round(dump.screen_height * scale),
|
|
1083
|
+
scale,
|
|
1084
|
+
rotation: dump.rotation,
|
|
1085
|
+
nodes: scaled.nodes
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
1088
|
+
}));
|
|
1089
|
+
}
|
|
1090
|
+
//#endregion
|
|
1091
|
+
//#region src/annotate.ts
|
|
1092
|
+
/**
|
|
1093
|
+
* Overlay a prominent tap marker on a PNG at device-native coordinates.
|
|
1094
|
+
*
|
|
1095
|
+
* The marker is a layered target: outer glow zone, white-bordered red ring,
|
|
1096
|
+
* solid center dot, and long crosshair arms — designed to be visible on any
|
|
1097
|
+
* background.
|
|
1098
|
+
*
|
|
1099
|
+
* @param png - the original PNG bytes (no scaling).
|
|
1100
|
+
* @param x - tap X in device pixels.
|
|
1101
|
+
* @param y - tap Y in device pixels.
|
|
1102
|
+
* @returns the annotated PNG bytes and dimensions.
|
|
1103
|
+
*/
|
|
1104
|
+
async function annotateTap(png, x, y) {
|
|
1105
|
+
const sharp = (await import("sharp")).default;
|
|
1106
|
+
let meta;
|
|
1107
|
+
try {
|
|
1108
|
+
meta = await sharp(png).metadata();
|
|
1109
|
+
} catch {
|
|
1110
|
+
throw new Error("cannot annotate: screenshot has no dimensions");
|
|
1111
|
+
}
|
|
1112
|
+
const width = meta.width ?? 0;
|
|
1113
|
+
const height = meta.height ?? 0;
|
|
1114
|
+
if (width === 0 || height === 0) throw new Error("cannot annotate: screenshot has no dimensions");
|
|
1115
|
+
const r = Math.max(28, Math.round(Math.min(width, height) * .04));
|
|
1116
|
+
const ringW = Math.max(4, Math.round(r * .14));
|
|
1117
|
+
const lineW = Math.max(3, Math.round(r * .1));
|
|
1118
|
+
const armLen = r * 2.4;
|
|
1119
|
+
const red = "#FF1744";
|
|
1120
|
+
const white = "#FFFFFF";
|
|
1121
|
+
const svg = `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg"><circle cx="${x}" cy="${y}" r="${Math.round(r * 2.2)}" fill="${red}" opacity="0.08"/><circle cx="${x}" cy="${y}" r="${r}" fill="none" stroke="${white}" stroke-width="${ringW + 4}" opacity="0.45"/><circle cx="${x}" cy="${y}" r="${r}" fill="none" stroke="${red}" stroke-width="${ringW}" opacity="0.9"/><circle cx="${x}" cy="${y}" r="${Math.round(r * .82)}" fill="none" stroke="${white}" stroke-width="2" opacity="0.5"/><line x1="${Math.round(x - armLen)}" y1="${y}" x2="${Math.round(x + armLen)}" y2="${y}" stroke="${white}" stroke-width="${lineW + 3}" opacity="0.35"/><line x1="${x}" y1="${Math.round(y - armLen)}" x2="${x}" y2="${Math.round(y + armLen)}" stroke="${white}" stroke-width="${lineW + 3}" opacity="0.35"/><line x1="${Math.round(x - armLen)}" y1="${y}" x2="${Math.round(x + armLen)}" y2="${y}" stroke="${red}" stroke-width="${lineW}" opacity="0.8"/><line x1="${x}" y1="${Math.round(y - armLen)}" x2="${x}" y2="${Math.round(y + armLen)}" stroke="${red}" stroke-width="${lineW}" opacity="0.8"/><circle cx="${x}" cy="${y}" r="${Math.round(r * .24)}" fill="none" stroke="${white}" stroke-width="2" opacity="0.6"/><circle cx="${x}" cy="${y}" r="${Math.round(r * .24)}" fill="${red}" opacity="0.85"/></svg>`;
|
|
1122
|
+
const buf = await sharp(png).composite([{
|
|
1123
|
+
input: Buffer.from(svg),
|
|
1124
|
+
top: 0,
|
|
1125
|
+
left: 0
|
|
1126
|
+
}]).png().toBuffer();
|
|
1127
|
+
return {
|
|
1128
|
+
data: new Uint8Array(buf),
|
|
1129
|
+
width,
|
|
1130
|
+
height
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
//#endregion
|
|
1134
|
+
//#region src/keys.ts
|
|
1135
|
+
/**
|
|
1136
|
+
* keys.ts — named Android keycode table for `android_press_key`.
|
|
1137
|
+
*
|
|
1138
|
+
* Maps human-readable key names to Android `KeyEvent` keycode integers.
|
|
1139
|
+
* The model may pass either a named key (`"home"`, `"back"`) or a bare
|
|
1140
|
+
* integer keycode; `resolveKey` normalizes both into a numeric keycode.
|
|
1141
|
+
*
|
|
1142
|
+
* @module @huanlin/dsh-plugin-android-use/src/keys
|
|
1143
|
+
*/
|
|
1144
|
+
/** Named key → Android keycode. Source: android.view.KeyEvent constants. */
|
|
1145
|
+
const KEY_CODES = {
|
|
1146
|
+
home: 3,
|
|
1147
|
+
back: 4,
|
|
1148
|
+
call: 5,
|
|
1149
|
+
endcall: 6,
|
|
1150
|
+
"0": 7,
|
|
1151
|
+
"1": 8,
|
|
1152
|
+
"2": 9,
|
|
1153
|
+
"3": 10,
|
|
1154
|
+
"4": 11,
|
|
1155
|
+
"5": 12,
|
|
1156
|
+
"6": 13,
|
|
1157
|
+
"7": 14,
|
|
1158
|
+
"8": 15,
|
|
1159
|
+
"9": 16,
|
|
1160
|
+
star: 17,
|
|
1161
|
+
pound: 18,
|
|
1162
|
+
dpad_up: 19,
|
|
1163
|
+
dpad_down: 20,
|
|
1164
|
+
dpad_left: 21,
|
|
1165
|
+
dpad_right: 22,
|
|
1166
|
+
dpad_center: 23,
|
|
1167
|
+
volume_up: 24,
|
|
1168
|
+
volume_down: 25,
|
|
1169
|
+
power: 26,
|
|
1170
|
+
camera: 27,
|
|
1171
|
+
clear: 28,
|
|
1172
|
+
a: 29,
|
|
1173
|
+
b: 30,
|
|
1174
|
+
c: 31,
|
|
1175
|
+
d: 32,
|
|
1176
|
+
e: 33,
|
|
1177
|
+
f: 34,
|
|
1178
|
+
g: 35,
|
|
1179
|
+
h: 36,
|
|
1180
|
+
i: 37,
|
|
1181
|
+
j: 38,
|
|
1182
|
+
k: 39,
|
|
1183
|
+
l: 40,
|
|
1184
|
+
m: 41,
|
|
1185
|
+
n: 42,
|
|
1186
|
+
o: 43,
|
|
1187
|
+
p: 44,
|
|
1188
|
+
q: 45,
|
|
1189
|
+
r: 46,
|
|
1190
|
+
s: 47,
|
|
1191
|
+
t: 48,
|
|
1192
|
+
u: 49,
|
|
1193
|
+
v: 50,
|
|
1194
|
+
w: 51,
|
|
1195
|
+
x: 52,
|
|
1196
|
+
y: 53,
|
|
1197
|
+
z: 54,
|
|
1198
|
+
comma: 55,
|
|
1199
|
+
period: 56,
|
|
1200
|
+
alt_left: 57,
|
|
1201
|
+
alt_right: 58,
|
|
1202
|
+
shift_left: 59,
|
|
1203
|
+
shift_right: 60,
|
|
1204
|
+
tab: 61,
|
|
1205
|
+
space: 62,
|
|
1206
|
+
sym: 63,
|
|
1207
|
+
explorer: 64,
|
|
1208
|
+
envelope: 65,
|
|
1209
|
+
enter: 66,
|
|
1210
|
+
del: 67,
|
|
1211
|
+
backspace: 67,
|
|
1212
|
+
grave: 68,
|
|
1213
|
+
minus: 69,
|
|
1214
|
+
equals: 70,
|
|
1215
|
+
left_bracket: 71,
|
|
1216
|
+
right_bracket: 72,
|
|
1217
|
+
backslash: 73,
|
|
1218
|
+
semicolon: 74,
|
|
1219
|
+
apostrophe: 75,
|
|
1220
|
+
slash: 76,
|
|
1221
|
+
at: 77,
|
|
1222
|
+
num: 78,
|
|
1223
|
+
headsethook: 79,
|
|
1224
|
+
focus: 80,
|
|
1225
|
+
plus: 81,
|
|
1226
|
+
menu: 82,
|
|
1227
|
+
notification: 83,
|
|
1228
|
+
search: 84,
|
|
1229
|
+
media_play_pause: 85,
|
|
1230
|
+
media_stop: 86,
|
|
1231
|
+
media_next: 87,
|
|
1232
|
+
media_previous: 88,
|
|
1233
|
+
media_rewind: 89,
|
|
1234
|
+
media_fast_forward: 90,
|
|
1235
|
+
mute: 91,
|
|
1236
|
+
page_up: 92,
|
|
1237
|
+
page_down: 93,
|
|
1238
|
+
pict_symbols: 94,
|
|
1239
|
+
switch_charset: 95,
|
|
1240
|
+
button_a: 96,
|
|
1241
|
+
button_b: 97,
|
|
1242
|
+
button_c: 98,
|
|
1243
|
+
button_x: 99,
|
|
1244
|
+
button_y: 100,
|
|
1245
|
+
button_z: 101,
|
|
1246
|
+
button_l1: 102,
|
|
1247
|
+
button_r1: 103,
|
|
1248
|
+
button_l2: 104,
|
|
1249
|
+
button_r2: 105,
|
|
1250
|
+
button_thumbl: 106,
|
|
1251
|
+
button_thumbr: 107,
|
|
1252
|
+
button_start: 108,
|
|
1253
|
+
button_select: 109,
|
|
1254
|
+
button_mode: 110,
|
|
1255
|
+
escape: 111,
|
|
1256
|
+
forward_del: 112,
|
|
1257
|
+
delete: 112,
|
|
1258
|
+
ctrl_left: 113,
|
|
1259
|
+
ctrl_right: 114,
|
|
1260
|
+
caps_lock: 115,
|
|
1261
|
+
scroll_lock: 116,
|
|
1262
|
+
meta_left: 117,
|
|
1263
|
+
meta_right: 118,
|
|
1264
|
+
function: 119,
|
|
1265
|
+
sysrq: 120,
|
|
1266
|
+
break: 121,
|
|
1267
|
+
move_home: 122,
|
|
1268
|
+
move_end: 123,
|
|
1269
|
+
insert: 124,
|
|
1270
|
+
forward: 125,
|
|
1271
|
+
media_play: 126,
|
|
1272
|
+
media_pause: 127,
|
|
1273
|
+
media_close: 128,
|
|
1274
|
+
media_eject: 129,
|
|
1275
|
+
media_record: 130,
|
|
1276
|
+
f1: 131,
|
|
1277
|
+
f2: 132,
|
|
1278
|
+
f3: 133,
|
|
1279
|
+
f4: 134,
|
|
1280
|
+
f5: 135,
|
|
1281
|
+
f6: 136,
|
|
1282
|
+
f7: 137,
|
|
1283
|
+
f8: 138,
|
|
1284
|
+
f9: 139,
|
|
1285
|
+
f10: 140,
|
|
1286
|
+
f11: 141,
|
|
1287
|
+
f12: 142,
|
|
1288
|
+
num_lock: 143,
|
|
1289
|
+
numpad_0: 144,
|
|
1290
|
+
numpad_1: 145,
|
|
1291
|
+
numpad_2: 146,
|
|
1292
|
+
numpad_3: 147,
|
|
1293
|
+
numpad_4: 148,
|
|
1294
|
+
numpad_5: 149,
|
|
1295
|
+
numpad_6: 150,
|
|
1296
|
+
numpad_7: 151,
|
|
1297
|
+
numpad_8: 152,
|
|
1298
|
+
numpad_9: 153,
|
|
1299
|
+
numpad_add: 154,
|
|
1300
|
+
numpad_subtract: 155,
|
|
1301
|
+
numpad_multiply: 156,
|
|
1302
|
+
numpad_divide: 157,
|
|
1303
|
+
numpad_dot: 158,
|
|
1304
|
+
numpad_comma: 159,
|
|
1305
|
+
numpad_enter: 160,
|
|
1306
|
+
numpad_equals: 161,
|
|
1307
|
+
numpad_left_paren: 162,
|
|
1308
|
+
numpad_right_paren: 163,
|
|
1309
|
+
volume_mute: 164,
|
|
1310
|
+
info: 165,
|
|
1311
|
+
channel_up: 166,
|
|
1312
|
+
channel_down: 167,
|
|
1313
|
+
zoom_in: 168,
|
|
1314
|
+
zoom_out: 169,
|
|
1315
|
+
tv: 170,
|
|
1316
|
+
window: 171,
|
|
1317
|
+
guide: 172,
|
|
1318
|
+
dvr: 173,
|
|
1319
|
+
bookmark: 174,
|
|
1320
|
+
captions: 175,
|
|
1321
|
+
settings: 176,
|
|
1322
|
+
tv_power: 177,
|
|
1323
|
+
tv_input: 178,
|
|
1324
|
+
stb_power: 179,
|
|
1325
|
+
stb_input: 180,
|
|
1326
|
+
avr_power: 181,
|
|
1327
|
+
avr_input: 182,
|
|
1328
|
+
prog_red: 183,
|
|
1329
|
+
prog_green: 184,
|
|
1330
|
+
prog_yellow: 185,
|
|
1331
|
+
prog_blue: 186,
|
|
1332
|
+
app_switch: 187,
|
|
1333
|
+
button_1: 188,
|
|
1334
|
+
button_2: 189,
|
|
1335
|
+
button_3: 190,
|
|
1336
|
+
button_4: 191,
|
|
1337
|
+
button_5: 192,
|
|
1338
|
+
button_6: 193,
|
|
1339
|
+
button_7: 194,
|
|
1340
|
+
button_8: 195,
|
|
1341
|
+
button_9: 196,
|
|
1342
|
+
button_10: 197,
|
|
1343
|
+
button_11: 198,
|
|
1344
|
+
button_12: 199,
|
|
1345
|
+
button_13: 200,
|
|
1346
|
+
button_14: 201,
|
|
1347
|
+
button_15: 202,
|
|
1348
|
+
button_16: 203,
|
|
1349
|
+
language_switch: 204,
|
|
1350
|
+
manner_mode: 205,
|
|
1351
|
+
"3d_mode": 206,
|
|
1352
|
+
contacts: 207,
|
|
1353
|
+
calendar: 208,
|
|
1354
|
+
music: 209,
|
|
1355
|
+
calculator: 210,
|
|
1356
|
+
zenkaku_hankaku: 211,
|
|
1357
|
+
eisu: 212,
|
|
1358
|
+
muhenkan: 213,
|
|
1359
|
+
henkan: 214,
|
|
1360
|
+
katakana_hiragana: 215,
|
|
1361
|
+
yen: 216,
|
|
1362
|
+
ro: 217,
|
|
1363
|
+
kana: 218,
|
|
1364
|
+
assist: 219,
|
|
1365
|
+
brightness_down: 220,
|
|
1366
|
+
brightness_up: 221,
|
|
1367
|
+
media_audio_track: 222,
|
|
1368
|
+
sleep: 223,
|
|
1369
|
+
wakeup: 224,
|
|
1370
|
+
soft_sleep: 225,
|
|
1371
|
+
cut: 277,
|
|
1372
|
+
copy: 278,
|
|
1373
|
+
paste: 279,
|
|
1374
|
+
system_navigation_up: 280,
|
|
1375
|
+
system_navigation_down: 281,
|
|
1376
|
+
system_navigation_left: 282,
|
|
1377
|
+
system_navigation_right: 283,
|
|
1378
|
+
all_apps: 284,
|
|
1379
|
+
refresh: 285
|
|
1380
|
+
};
|
|
1381
|
+
/**
|
|
1382
|
+
* Resolve a key name or raw keycode into a numeric Android keycode.
|
|
1383
|
+
* @param input - named key (case-insensitive, e.g. `"home"`, `"BACK"`) or a
|
|
1384
|
+
* string/number representing a raw keycode integer.
|
|
1385
|
+
* @returns the numeric keycode.
|
|
1386
|
+
* @throws when the name is unknown and the input is not a valid integer.
|
|
1387
|
+
*/
|
|
1388
|
+
function resolveKey(input) {
|
|
1389
|
+
if (typeof input === "number") {
|
|
1390
|
+
if (!Number.isInteger(input)) throw new Error(`keycode must be an integer, got ${input}`);
|
|
1391
|
+
return input;
|
|
1392
|
+
}
|
|
1393
|
+
const asNum = Number(input);
|
|
1394
|
+
if (input.trim() !== "" && Number.isInteger(asNum) && String(asNum) === input.trim()) return asNum;
|
|
1395
|
+
const lower = input.toLowerCase();
|
|
1396
|
+
const code = KEY_CODES[lower];
|
|
1397
|
+
if (code === void 0) throw new Error(`unknown key name "${input}"; pass a named key (e.g. "home", "back") or a raw integer keycode`);
|
|
1398
|
+
return code;
|
|
1399
|
+
}
|
|
1400
|
+
//#endregion
|
|
1401
|
+
//#region src/tools/input.ts
|
|
1402
|
+
/**
|
|
1403
|
+
* Escape text for `adb shell input text`. Spaces become `%s` and shell
|
|
1404
|
+
* metacharacters are backslash-escaped. Non-ASCII characters are rejected
|
|
1405
|
+
* (the `input` command cannot encode them).
|
|
1406
|
+
* @param text - the raw text to escape.
|
|
1407
|
+
* @returns the escaped text safe for `adb shell input text`.
|
|
1408
|
+
* @throws when the text contains non-ASCII characters.
|
|
1409
|
+
*/
|
|
1410
|
+
function escapeInputText(text) {
|
|
1411
|
+
for (let i = 0; i < text.length; i++) if (text.charCodeAt(i) > 127) throw new Error(`the "input" text mode cannot encode non-ASCII character "${text[i]}" (U+${text.charCodeAt(i).toString(16).toUpperCase()}); set the plugin config inputTextMode to "adbkeyboard" to inject Unicode text via ADBKeyboard`);
|
|
1412
|
+
let out = "";
|
|
1413
|
+
for (let i = 0; i < text.length; i++) {
|
|
1414
|
+
const ch = text[i];
|
|
1415
|
+
switch (ch) {
|
|
1416
|
+
case " ":
|
|
1417
|
+
out += "%s";
|
|
1418
|
+
break;
|
|
1419
|
+
case "&":
|
|
1420
|
+
case "<":
|
|
1421
|
+
case ">":
|
|
1422
|
+
case ";":
|
|
1423
|
+
case "(":
|
|
1424
|
+
case ")":
|
|
1425
|
+
case "|":
|
|
1426
|
+
case "^":
|
|
1427
|
+
case "*":
|
|
1428
|
+
case "~":
|
|
1429
|
+
case "\"":
|
|
1430
|
+
case "'":
|
|
1431
|
+
case "`":
|
|
1432
|
+
case "$":
|
|
1433
|
+
case "!":
|
|
1434
|
+
case "#":
|
|
1435
|
+
out += "\\" + ch;
|
|
1436
|
+
break;
|
|
1437
|
+
case "\\":
|
|
1438
|
+
out += "\\\\";
|
|
1439
|
+
break;
|
|
1440
|
+
default: out += ch;
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
return out;
|
|
1444
|
+
}
|
|
1445
|
+
/**
|
|
1446
|
+
* URL-encode text for ADBKeyboard broadcast injection (`am broadcast -a ADB_INPUT_TEXT --es msg <encoded>`).
|
|
1447
|
+
* @param text - the raw text to encode (any Unicode).
|
|
1448
|
+
* @returns the percent-encoded text.
|
|
1449
|
+
*/
|
|
1450
|
+
function encodeAdbKeyboard(text) {
|
|
1451
|
+
return encodeURIComponent(text);
|
|
1452
|
+
}
|
|
1453
|
+
function renderTap(_args, value) {
|
|
1454
|
+
const v = value;
|
|
1455
|
+
const lines = [`Tap at image (${v.x}, ${v.y}) → device (${v.device_x}, ${v.device_y}) on ${v.serial}: ${v.action} ×${v.times}` + (v.duration_ms > 0 ? ` (${v.duration_ms}ms)` : "")];
|
|
1456
|
+
if (v.pre_tap_screenshot !== null) lines.push(`Pre-tap screenshot (annotated with tap position marker): ${v.pre_tap_screenshot.width}x${v.pre_tap_screenshot.height} px, ${v.pre_tap_screenshot.bytes} bytes.`);
|
|
1457
|
+
else lines.push("No pre-tap screenshot (attachment store unavailable or capture failed).");
|
|
1458
|
+
if (v.post_tap_screenshot !== null) lines.push(`Post-tap screenshot (showing the screen result after tap): ${v.post_tap_screenshot.width}x${v.post_tap_screenshot.height} px, ${v.post_tap_screenshot.bytes} bytes.`);
|
|
1459
|
+
else lines.push("No post-tap screenshot (attachment store unavailable or capture failed).");
|
|
1460
|
+
lines.push(`Images emitted to model: ${v.screenshot_emitted ? "yes" : "no"}.`);
|
|
1461
|
+
const blocks = [{
|
|
1462
|
+
type: "text",
|
|
1463
|
+
text: lines.join("\n")
|
|
1464
|
+
}];
|
|
1465
|
+
if (v.screenshot_emitted) {
|
|
1466
|
+
if (v.pre_tap_screenshot !== null) {
|
|
1467
|
+
blocks.push({
|
|
1468
|
+
type: "text",
|
|
1469
|
+
text: "Pre-tap screenshot (annotated with tap position marker):"
|
|
1470
|
+
});
|
|
1471
|
+
blocks.push({
|
|
1472
|
+
type: "image",
|
|
1473
|
+
attachment: {
|
|
1474
|
+
attachmentId: v.pre_tap_screenshot.attachmentId,
|
|
1475
|
+
mediaType: "image/png",
|
|
1476
|
+
bytes: v.pre_tap_screenshot.bytes,
|
|
1477
|
+
width: v.pre_tap_screenshot.width,
|
|
1478
|
+
height: v.pre_tap_screenshot.height
|
|
1479
|
+
}
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
if (v.post_tap_screenshot !== null) {
|
|
1483
|
+
blocks.push({
|
|
1484
|
+
type: "text",
|
|
1485
|
+
text: "Post-tap screenshot (showing the screen result after tap):"
|
|
1486
|
+
});
|
|
1487
|
+
blocks.push({
|
|
1488
|
+
type: "image",
|
|
1489
|
+
attachment: {
|
|
1490
|
+
attachmentId: v.post_tap_screenshot.attachmentId,
|
|
1491
|
+
mediaType: "image/png",
|
|
1492
|
+
bytes: v.post_tap_screenshot.bytes,
|
|
1493
|
+
width: v.post_tap_screenshot.width,
|
|
1494
|
+
height: v.post_tap_screenshot.height
|
|
1495
|
+
}
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
return blocks;
|
|
1500
|
+
}
|
|
1501
|
+
function renderSwipe(value) {
|
|
1502
|
+
return `Swipe image (${value.x1}, ${value.y1}) → (${value.x2}, ${value.y2}) / device (${value.device_x1}, ${value.device_y1}) → (${value.device_x2}, ${value.device_y2}) on ${value.serial}` + (value.duration_ms > 0 ? ` over ${value.duration_ms}ms` : "");
|
|
1503
|
+
}
|
|
1504
|
+
function renderPressKey(value) {
|
|
1505
|
+
return `Pressed key "${value.key}" (keycode ${value.keycode}) on ${value.serial} ×${value.times}`;
|
|
1506
|
+
}
|
|
1507
|
+
function renderInputText(value) {
|
|
1508
|
+
const lines = [`Input text on ${value.serial} (mode: ${value.mode}):`, ` text: ${JSON.stringify(value.text)}`];
|
|
1509
|
+
if (value.submitted) lines.push(" submitted (Enter pressed)");
|
|
1510
|
+
return lines.join("\n");
|
|
1511
|
+
}
|
|
1512
|
+
function textRender$1(fn) {
|
|
1513
|
+
return (_args, value) => [{
|
|
1514
|
+
type: "text",
|
|
1515
|
+
text: fn(value)
|
|
1516
|
+
}];
|
|
1517
|
+
}
|
|
1518
|
+
/** Register `android_tap`, `android_swipe`, `android_press_key`, `android_input_text`. */
|
|
1519
|
+
function registerInputTools(ctx, deps) {
|
|
1520
|
+
ctx.tools.register(defineTool({
|
|
1521
|
+
name: "android_tap",
|
|
1522
|
+
description: "Tap a point on the Android screen. Pass (x, y) in screenshot image coordinates — the same coordinate space as the pixels in android_screenshot and the \"center\" values from android_ui_dump. The plugin automatically converts these to device-native coordinates for execution. Use `duration_ms` for a long-press (hold). Use `times` to repeat the tap. A pre-tap screenshot annotated with a marker at the tap position is captured and returned when an attachment store is available.",
|
|
1523
|
+
parameters: {
|
|
1524
|
+
x: {
|
|
1525
|
+
type: "integer",
|
|
1526
|
+
required: true,
|
|
1527
|
+
description: "X coordinate in screenshot image space."
|
|
1528
|
+
},
|
|
1529
|
+
y: {
|
|
1530
|
+
type: "integer",
|
|
1531
|
+
required: true,
|
|
1532
|
+
description: "Y coordinate in screenshot image space."
|
|
1533
|
+
},
|
|
1534
|
+
duration_ms: {
|
|
1535
|
+
type: "integer",
|
|
1536
|
+
description: "Hold duration in milliseconds. When > 0, performs a long-press (swipe-to-same-point) instead of a quick tap."
|
|
1537
|
+
},
|
|
1538
|
+
times: {
|
|
1539
|
+
type: "integer",
|
|
1540
|
+
description: "Number of times to repeat the tap (default 1)."
|
|
1541
|
+
},
|
|
1542
|
+
serial: {
|
|
1543
|
+
type: "string",
|
|
1544
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
1545
|
+
}
|
|
1546
|
+
},
|
|
1547
|
+
output: {
|
|
1548
|
+
schema: {
|
|
1549
|
+
type: "object",
|
|
1550
|
+
additionalProperties: false,
|
|
1551
|
+
properties: {
|
|
1552
|
+
serial: {
|
|
1553
|
+
type: "string",
|
|
1554
|
+
required: true
|
|
1555
|
+
},
|
|
1556
|
+
x: {
|
|
1557
|
+
type: "integer",
|
|
1558
|
+
required: true
|
|
1559
|
+
},
|
|
1560
|
+
y: {
|
|
1561
|
+
type: "integer",
|
|
1562
|
+
required: true
|
|
1563
|
+
},
|
|
1564
|
+
device_x: {
|
|
1565
|
+
type: "integer",
|
|
1566
|
+
required: true
|
|
1567
|
+
},
|
|
1568
|
+
device_y: {
|
|
1569
|
+
type: "integer",
|
|
1570
|
+
required: true
|
|
1571
|
+
},
|
|
1572
|
+
duration_ms: {
|
|
1573
|
+
type: "integer",
|
|
1574
|
+
required: true
|
|
1575
|
+
},
|
|
1576
|
+
times: {
|
|
1577
|
+
type: "integer",
|
|
1578
|
+
required: true
|
|
1579
|
+
},
|
|
1580
|
+
action: {
|
|
1581
|
+
type: "string",
|
|
1582
|
+
enum: ["tap", "long_press"],
|
|
1583
|
+
required: true
|
|
1584
|
+
},
|
|
1585
|
+
pre_tap_screenshot: {
|
|
1586
|
+
oneOf: [{ type: "null" }, {
|
|
1587
|
+
type: "object",
|
|
1588
|
+
additionalProperties: false,
|
|
1589
|
+
properties: {
|
|
1590
|
+
attachmentId: {
|
|
1591
|
+
type: "string",
|
|
1592
|
+
required: true
|
|
1593
|
+
},
|
|
1594
|
+
bytes: {
|
|
1595
|
+
type: "integer",
|
|
1596
|
+
required: true
|
|
1597
|
+
},
|
|
1598
|
+
width: {
|
|
1599
|
+
type: "integer",
|
|
1600
|
+
required: true
|
|
1601
|
+
},
|
|
1602
|
+
height: {
|
|
1603
|
+
type: "integer",
|
|
1604
|
+
required: true
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
}],
|
|
1608
|
+
required: true
|
|
1609
|
+
},
|
|
1610
|
+
post_tap_screenshot: {
|
|
1611
|
+
oneOf: [{ type: "null" }, {
|
|
1612
|
+
type: "object",
|
|
1613
|
+
additionalProperties: false,
|
|
1614
|
+
properties: {
|
|
1615
|
+
attachmentId: {
|
|
1616
|
+
type: "string",
|
|
1617
|
+
required: true
|
|
1618
|
+
},
|
|
1619
|
+
bytes: {
|
|
1620
|
+
type: "integer",
|
|
1621
|
+
required: true
|
|
1622
|
+
},
|
|
1623
|
+
width: {
|
|
1624
|
+
type: "integer",
|
|
1625
|
+
required: true
|
|
1626
|
+
},
|
|
1627
|
+
height: {
|
|
1628
|
+
type: "integer",
|
|
1629
|
+
required: true
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
}],
|
|
1633
|
+
required: true
|
|
1634
|
+
},
|
|
1635
|
+
screenshot_emitted: {
|
|
1636
|
+
type: "boolean",
|
|
1637
|
+
required: true
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
},
|
|
1641
|
+
render: renderTap
|
|
1642
|
+
},
|
|
1643
|
+
async execute(args, exec) {
|
|
1644
|
+
const a = args;
|
|
1645
|
+
const config = deps.getConfig();
|
|
1646
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
1647
|
+
const dur = typeof a.duration_ms === "number" && a.duration_ms > 0 ? a.duration_ms : 0;
|
|
1648
|
+
const times = typeof a.times === "number" && a.times > 0 ? a.times : 1;
|
|
1649
|
+
const action = dur > 0 ? "long_press" : "tap";
|
|
1650
|
+
let preScreenshot = null;
|
|
1651
|
+
let postScreenshot = null;
|
|
1652
|
+
let screenshotEmitted = false;
|
|
1653
|
+
const attachments = ctx.get("attachments");
|
|
1654
|
+
let scaleX = 1;
|
|
1655
|
+
let scaleY = 1;
|
|
1656
|
+
if (attachments !== void 0) try {
|
|
1657
|
+
const raw = await deps.adb.execOutBinary(serial, "screencap -p", exec.signal);
|
|
1658
|
+
const fitted = await fitToMaxDimension(new Uint8Array(raw), attachments.imageLimits.maxImageDimension);
|
|
1659
|
+
scaleX = fitted.scaleX;
|
|
1660
|
+
scaleY = fitted.scaleY;
|
|
1661
|
+
const annotated = await annotateTap(fitted.data, a.x, a.y);
|
|
1662
|
+
const ref = await attachments.saveImage({
|
|
1663
|
+
data: annotated.data,
|
|
1664
|
+
mediaType: "image/png",
|
|
1665
|
+
name: "pre_tap_screenshot.png"
|
|
1666
|
+
});
|
|
1667
|
+
preScreenshot = {
|
|
1668
|
+
attachmentId: ref.attachmentId,
|
|
1669
|
+
bytes: ref.bytes,
|
|
1670
|
+
width: ref.width,
|
|
1671
|
+
height: ref.height
|
|
1672
|
+
};
|
|
1673
|
+
} catch {}
|
|
1674
|
+
const deviceX = Math.round(a.x / scaleX);
|
|
1675
|
+
const deviceY = Math.round(a.y / scaleY);
|
|
1676
|
+
for (let i = 0; i < times; i++) if (dur > 0) await deps.adb.shell(serial, `input swipe ${deviceX} ${deviceY} ${deviceX} ${deviceY} ${dur}`, exec.signal);
|
|
1677
|
+
else await deps.adb.shell(serial, `input tap ${deviceX} ${deviceY}`, exec.signal);
|
|
1678
|
+
if (attachments !== void 0) try {
|
|
1679
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
1680
|
+
const raw = await deps.adb.execOutBinary(serial, "screencap -p", exec.signal);
|
|
1681
|
+
const fitted = await fitToMaxDimension(new Uint8Array(raw), attachments.imageLimits.maxImageDimension);
|
|
1682
|
+
const ref = await attachments.saveImage({
|
|
1683
|
+
data: fitted.data,
|
|
1684
|
+
mediaType: "image/png",
|
|
1685
|
+
name: "post_tap_screenshot.png"
|
|
1686
|
+
});
|
|
1687
|
+
postScreenshot = {
|
|
1688
|
+
attachmentId: ref.attachmentId,
|
|
1689
|
+
bytes: ref.bytes,
|
|
1690
|
+
width: ref.width,
|
|
1691
|
+
height: ref.height
|
|
1692
|
+
};
|
|
1693
|
+
screenshotEmitted = await routeIsImageCapable(ctx, exec, exec.signal);
|
|
1694
|
+
} catch {}
|
|
1695
|
+
return {
|
|
1696
|
+
serial,
|
|
1697
|
+
x: a.x,
|
|
1698
|
+
y: a.y,
|
|
1699
|
+
device_x: deviceX,
|
|
1700
|
+
device_y: deviceY,
|
|
1701
|
+
duration_ms: dur,
|
|
1702
|
+
times,
|
|
1703
|
+
action,
|
|
1704
|
+
pre_tap_screenshot: preScreenshot,
|
|
1705
|
+
post_tap_screenshot: postScreenshot,
|
|
1706
|
+
screenshot_emitted: screenshotEmitted
|
|
1707
|
+
};
|
|
1708
|
+
}
|
|
1709
|
+
}));
|
|
1710
|
+
ctx.tools.register(defineTool({
|
|
1711
|
+
name: "android_swipe",
|
|
1712
|
+
description: "Swipe from one point to another on the Android screen. Pass start and end coordinates in screenshot image space (the same coordinate space as android_screenshot and android_ui_dump). The plugin automatically converts these to device-native coordinates for execution. Use `duration_ms` to control swipe speed (longer = slower).",
|
|
1713
|
+
parameters: {
|
|
1714
|
+
x1: {
|
|
1715
|
+
type: "integer",
|
|
1716
|
+
required: true,
|
|
1717
|
+
description: "Start X coordinate in screenshot image space."
|
|
1718
|
+
},
|
|
1719
|
+
y1: {
|
|
1720
|
+
type: "integer",
|
|
1721
|
+
required: true,
|
|
1722
|
+
description: "Start Y coordinate in screenshot image space."
|
|
1723
|
+
},
|
|
1724
|
+
x2: {
|
|
1725
|
+
type: "integer",
|
|
1726
|
+
required: true,
|
|
1727
|
+
description: "End X coordinate in screenshot image space."
|
|
1728
|
+
},
|
|
1729
|
+
y2: {
|
|
1730
|
+
type: "integer",
|
|
1731
|
+
required: true,
|
|
1732
|
+
description: "End Y coordinate in screenshot image space."
|
|
1733
|
+
},
|
|
1734
|
+
duration_ms: {
|
|
1735
|
+
type: "integer",
|
|
1736
|
+
description: "Swipe duration in milliseconds (default 300). Longer values produce slower swipes."
|
|
1737
|
+
},
|
|
1738
|
+
serial: {
|
|
1739
|
+
type: "string",
|
|
1740
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
1741
|
+
}
|
|
1742
|
+
},
|
|
1743
|
+
output: {
|
|
1744
|
+
schema: {
|
|
1745
|
+
type: "object",
|
|
1746
|
+
additionalProperties: false,
|
|
1747
|
+
properties: {
|
|
1748
|
+
serial: {
|
|
1749
|
+
type: "string",
|
|
1750
|
+
required: true
|
|
1751
|
+
},
|
|
1752
|
+
x1: {
|
|
1753
|
+
type: "integer",
|
|
1754
|
+
required: true
|
|
1755
|
+
},
|
|
1756
|
+
y1: {
|
|
1757
|
+
type: "integer",
|
|
1758
|
+
required: true
|
|
1759
|
+
},
|
|
1760
|
+
x2: {
|
|
1761
|
+
type: "integer",
|
|
1762
|
+
required: true
|
|
1763
|
+
},
|
|
1764
|
+
y2: {
|
|
1765
|
+
type: "integer",
|
|
1766
|
+
required: true
|
|
1767
|
+
},
|
|
1768
|
+
device_x1: {
|
|
1769
|
+
type: "integer",
|
|
1770
|
+
required: true
|
|
1771
|
+
},
|
|
1772
|
+
device_y1: {
|
|
1773
|
+
type: "integer",
|
|
1774
|
+
required: true
|
|
1775
|
+
},
|
|
1776
|
+
device_x2: {
|
|
1777
|
+
type: "integer",
|
|
1778
|
+
required: true
|
|
1779
|
+
},
|
|
1780
|
+
device_y2: {
|
|
1781
|
+
type: "integer",
|
|
1782
|
+
required: true
|
|
1783
|
+
},
|
|
1784
|
+
duration_ms: {
|
|
1785
|
+
type: "integer",
|
|
1786
|
+
required: true
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
},
|
|
1790
|
+
render: textRender$1(renderSwipe)
|
|
1791
|
+
},
|
|
1792
|
+
async execute(args, exec) {
|
|
1793
|
+
const a = args;
|
|
1794
|
+
const config = deps.getConfig();
|
|
1795
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
1796
|
+
const dur = typeof a.duration_ms === "number" && a.duration_ms > 0 ? a.duration_ms : 300;
|
|
1797
|
+
const attachments = ctx.get("attachments");
|
|
1798
|
+
let scale = 1;
|
|
1799
|
+
if (attachments !== void 0) {
|
|
1800
|
+
const { width: devW, height: devH } = parseWmSize$1(await deps.adb.shell(serial, "wm size", exec.signal));
|
|
1801
|
+
scale = computeScale(devW, devH, attachments.imageLimits.maxImageDimension);
|
|
1802
|
+
}
|
|
1803
|
+
const dx1 = Math.round(a.x1 / scale);
|
|
1804
|
+
const dy1 = Math.round(a.y1 / scale);
|
|
1805
|
+
const dx2 = Math.round(a.x2 / scale);
|
|
1806
|
+
const dy2 = Math.round(a.y2 / scale);
|
|
1807
|
+
await deps.adb.shell(serial, `input swipe ${dx1} ${dy1} ${dx2} ${dy2} ${dur}`, exec.signal);
|
|
1808
|
+
return {
|
|
1809
|
+
serial,
|
|
1810
|
+
x1: a.x1,
|
|
1811
|
+
y1: a.y1,
|
|
1812
|
+
x2: a.x2,
|
|
1813
|
+
y2: a.y2,
|
|
1814
|
+
device_x1: dx1,
|
|
1815
|
+
device_y1: dy1,
|
|
1816
|
+
device_x2: dx2,
|
|
1817
|
+
device_y2: dy2,
|
|
1818
|
+
duration_ms: dur
|
|
1819
|
+
};
|
|
1820
|
+
}
|
|
1821
|
+
}));
|
|
1822
|
+
ctx.tools.register(defineTool({
|
|
1823
|
+
name: "android_press_key",
|
|
1824
|
+
description: "Press a hardware/key event key on the Android device. Pass a named key (case-insensitive: \"home\", \"back\", \"app_switch\", \"power\", \"enter\", \"volume_up\", \"volume_down\", \"mute\", \"camera\", \"search\", \"menu\", \"escape\", \"delete\", \"tab\", \"space\", \"dpad_up\", \"dpad_down\", \"dpad_left\", \"dpad_right\", \"dpad_center\", etc.) or a raw integer keycode. Use `times` to repeat.",
|
|
1825
|
+
parameters: {
|
|
1826
|
+
key: {
|
|
1827
|
+
oneOf: [{
|
|
1828
|
+
type: "string",
|
|
1829
|
+
description: "Named key (case-insensitive), e.g. \"home\", \"back\", \"app_switch\"."
|
|
1830
|
+
}, {
|
|
1831
|
+
type: "integer",
|
|
1832
|
+
description: "Raw Android keycode integer."
|
|
1833
|
+
}],
|
|
1834
|
+
required: true,
|
|
1835
|
+
description: "Key to press: a named key or a raw integer keycode."
|
|
1836
|
+
},
|
|
1837
|
+
times: {
|
|
1838
|
+
type: "integer",
|
|
1839
|
+
description: "Number of times to repeat the key press (default 1)."
|
|
1840
|
+
},
|
|
1841
|
+
serial: {
|
|
1842
|
+
type: "string",
|
|
1843
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
1844
|
+
}
|
|
1845
|
+
},
|
|
1846
|
+
output: {
|
|
1847
|
+
schema: {
|
|
1848
|
+
type: "object",
|
|
1849
|
+
additionalProperties: false,
|
|
1850
|
+
properties: {
|
|
1851
|
+
serial: {
|
|
1852
|
+
type: "string",
|
|
1853
|
+
required: true
|
|
1854
|
+
},
|
|
1855
|
+
key: {
|
|
1856
|
+
type: "string",
|
|
1857
|
+
required: true
|
|
1858
|
+
},
|
|
1859
|
+
keycode: {
|
|
1860
|
+
type: "integer",
|
|
1861
|
+
required: true
|
|
1862
|
+
},
|
|
1863
|
+
times: {
|
|
1864
|
+
type: "integer",
|
|
1865
|
+
required: true
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
},
|
|
1869
|
+
render: textRender$1(renderPressKey)
|
|
1870
|
+
},
|
|
1871
|
+
async execute(args, exec) {
|
|
1872
|
+
const a = args;
|
|
1873
|
+
const config = deps.getConfig();
|
|
1874
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
1875
|
+
const keycode = resolveKey(a.key);
|
|
1876
|
+
const times = typeof a.times === "number" && a.times > 0 ? a.times : 1;
|
|
1877
|
+
const keyStr = typeof a.key === "number" ? String(a.key) : a.key;
|
|
1878
|
+
for (let i = 0; i < times; i++) await deps.adb.shell(serial, `input keyevent ${keycode}`, exec.signal);
|
|
1879
|
+
return {
|
|
1880
|
+
serial,
|
|
1881
|
+
key: keyStr,
|
|
1882
|
+
keycode,
|
|
1883
|
+
times
|
|
1884
|
+
};
|
|
1885
|
+
}
|
|
1886
|
+
}));
|
|
1887
|
+
ctx.tools.register(defineTool({
|
|
1888
|
+
name: "android_input_text",
|
|
1889
|
+
description: "Type text into the focused input field on the Android device. In \"input\" mode (default), only ASCII text is supported — non-ASCII characters require switching the plugin config `inputTextMode` to \"adbkeyboard\" (requires the ADBKeyboard IME installed on the device). Pass `submit: true` to press Enter after typing.",
|
|
1890
|
+
parameters: {
|
|
1891
|
+
text: {
|
|
1892
|
+
type: "string",
|
|
1893
|
+
required: true,
|
|
1894
|
+
description: "Text to type into the focused field."
|
|
1895
|
+
},
|
|
1896
|
+
submit: {
|
|
1897
|
+
type: "boolean",
|
|
1898
|
+
description: "When true, presses Enter (keycode 66) after typing the text."
|
|
1899
|
+
},
|
|
1900
|
+
serial: {
|
|
1901
|
+
type: "string",
|
|
1902
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
1903
|
+
}
|
|
1904
|
+
},
|
|
1905
|
+
output: {
|
|
1906
|
+
schema: {
|
|
1907
|
+
type: "object",
|
|
1908
|
+
additionalProperties: false,
|
|
1909
|
+
properties: {
|
|
1910
|
+
serial: {
|
|
1911
|
+
type: "string",
|
|
1912
|
+
required: true
|
|
1913
|
+
},
|
|
1914
|
+
text: {
|
|
1915
|
+
type: "string",
|
|
1916
|
+
required: true
|
|
1917
|
+
},
|
|
1918
|
+
mode: {
|
|
1919
|
+
type: "string",
|
|
1920
|
+
enum: ["input", "adbkeyboard"],
|
|
1921
|
+
required: true
|
|
1922
|
+
},
|
|
1923
|
+
submitted: {
|
|
1924
|
+
type: "boolean",
|
|
1925
|
+
required: true
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
},
|
|
1929
|
+
render: textRender$1(renderInputText)
|
|
1930
|
+
},
|
|
1931
|
+
async execute(args, exec) {
|
|
1932
|
+
const a = args;
|
|
1933
|
+
const config = deps.getConfig();
|
|
1934
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
1935
|
+
const mode = config.inputTextMode;
|
|
1936
|
+
const submitted = a.submit === true;
|
|
1937
|
+
if (mode === "adbkeyboard") {
|
|
1938
|
+
const encoded = encodeAdbKeyboard(a.text);
|
|
1939
|
+
await deps.adb.shell(serial, `am broadcast -a ADB_INPUT_TEXT --es msg "${encoded}"`, exec.signal);
|
|
1940
|
+
} else {
|
|
1941
|
+
const escaped = escapeInputText(a.text);
|
|
1942
|
+
await deps.adb.shell(serial, `input text "${escaped}"`, exec.signal);
|
|
1943
|
+
}
|
|
1944
|
+
if (submitted) await deps.adb.shell(serial, "input keyevent 66", exec.signal);
|
|
1945
|
+
return {
|
|
1946
|
+
serial,
|
|
1947
|
+
text: a.text,
|
|
1948
|
+
mode,
|
|
1949
|
+
submitted
|
|
1950
|
+
};
|
|
1951
|
+
}
|
|
1952
|
+
}));
|
|
1953
|
+
}
|
|
1954
|
+
//#endregion
|
|
1955
|
+
//#region src/tools/apps.ts
|
|
1956
|
+
/**
|
|
1957
|
+
* Parse `dumpsys window` output for the current focus window.
|
|
1958
|
+
*
|
|
1959
|
+
* Example line: ` mCurrentFocus=Window{f6144b2 u0 com.android.launcher/com.android.launcher.Launcher}`
|
|
1960
|
+
* Also handles `mCurrentFocus=null` (display off or no focused window).
|
|
1961
|
+
* @param output - the grep-filtered dumpsys window output.
|
|
1962
|
+
* @returns the package and activity, or null when no window is focused.
|
|
1963
|
+
*/
|
|
1964
|
+
function parseForegroundApp(output) {
|
|
1965
|
+
const lines = output.split(/\r?\n/);
|
|
1966
|
+
let lastFocus = null;
|
|
1967
|
+
for (const line of lines) {
|
|
1968
|
+
const match = /mCurrentFocus=(.+)/.exec(line.trim());
|
|
1969
|
+
if (match !== null) lastFocus = match[1];
|
|
1970
|
+
}
|
|
1971
|
+
if (lastFocus === null) return null;
|
|
1972
|
+
if (lastFocus === "null") return null;
|
|
1973
|
+
const windowMatch = /Window\{[^}]*?\s+([^/\s}]+)\/([^\s}]+)\s*\}/.exec(lastFocus);
|
|
1974
|
+
if (windowMatch !== null) return {
|
|
1975
|
+
package: windowMatch[1],
|
|
1976
|
+
activity: windowMatch[2],
|
|
1977
|
+
windowTitle: null
|
|
1978
|
+
};
|
|
1979
|
+
const pairMatch = /([a-zA-Z0-9_.]+)\/([a-zA-Z0-9_.]+)/.exec(lastFocus);
|
|
1980
|
+
if (pairMatch !== null) return {
|
|
1981
|
+
package: pairMatch[1],
|
|
1982
|
+
activity: pairMatch[2],
|
|
1983
|
+
windowTitle: null
|
|
1984
|
+
};
|
|
1985
|
+
return {
|
|
1986
|
+
package: lastFocus,
|
|
1987
|
+
activity: "",
|
|
1988
|
+
windowTitle: lastFocus
|
|
1989
|
+
};
|
|
1990
|
+
}
|
|
1991
|
+
/** Parse `dumpsys power` output for the wakefulness state. */
|
|
1992
|
+
function parseWakefulness(output) {
|
|
1993
|
+
return /mWakefulness=Awake/.test(output);
|
|
1994
|
+
}
|
|
1995
|
+
function renderOpenApp(value) {
|
|
1996
|
+
return `Opened app ${value.activity !== void 0 ? `${value.package}/${value.activity}` : value.package} on ${value.serial}: ${value.started ? "started" : "failed"}`;
|
|
1997
|
+
}
|
|
1998
|
+
function renderForegroundApp(value) {
|
|
1999
|
+
if (value.package === null) return `No focused window on ${value.serial} (screen on: ${value.screen_on}).`;
|
|
2000
|
+
const lines = [
|
|
2001
|
+
`Foreground app on ${value.serial}:`,
|
|
2002
|
+
` package: ${value.package}`,
|
|
2003
|
+
` activity: ${value.activity ?? "(unknown)"}`
|
|
2004
|
+
];
|
|
2005
|
+
if (value.window_title !== null) lines.push(` window: ${value.window_title}`);
|
|
2006
|
+
lines.push(` screen on: ${value.screen_on}`);
|
|
2007
|
+
return lines.join("\n");
|
|
2008
|
+
}
|
|
2009
|
+
function textRender(fn) {
|
|
2010
|
+
return (_args, value) => [{
|
|
2011
|
+
type: "text",
|
|
2012
|
+
text: fn(value)
|
|
2013
|
+
}];
|
|
2014
|
+
}
|
|
2015
|
+
/** Register `android_open_app` and `android_foreground_app`. */
|
|
2016
|
+
function registerAppTools(ctx, deps) {
|
|
2017
|
+
ctx.tools.register(defineTool({
|
|
2018
|
+
name: "android_open_app",
|
|
2019
|
+
description: "Open an app on the Android device by package name. If you know the specific activity, pass it; otherwise the app's default launcher activity is started via monkey. Example packages: \"com.android.settings\", \"com.android.chrome\", \"com.tencent.mm\" (WeChat).",
|
|
2020
|
+
parameters: {
|
|
2021
|
+
package: {
|
|
2022
|
+
type: "string",
|
|
2023
|
+
required: true,
|
|
2024
|
+
description: "Android package name, e.g. \"com.android.settings\"."
|
|
2025
|
+
},
|
|
2026
|
+
activity: {
|
|
2027
|
+
type: "string",
|
|
2028
|
+
description: "Specific activity to start (e.g. \".Settings\"). Omit to launch the app's default activity."
|
|
2029
|
+
}
|
|
2030
|
+
},
|
|
2031
|
+
output: {
|
|
2032
|
+
schema: {
|
|
2033
|
+
type: "object",
|
|
2034
|
+
additionalProperties: false,
|
|
2035
|
+
properties: {
|
|
2036
|
+
serial: {
|
|
2037
|
+
type: "string",
|
|
2038
|
+
required: true
|
|
2039
|
+
},
|
|
2040
|
+
package: {
|
|
2041
|
+
type: "string",
|
|
2042
|
+
required: true
|
|
2043
|
+
},
|
|
2044
|
+
activity: { type: "string" },
|
|
2045
|
+
started: {
|
|
2046
|
+
type: "boolean",
|
|
2047
|
+
required: true
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
},
|
|
2051
|
+
render: textRender(renderOpenApp)
|
|
2052
|
+
},
|
|
2053
|
+
async execute(args, exec) {
|
|
2054
|
+
const a = args;
|
|
2055
|
+
const config = deps.getConfig();
|
|
2056
|
+
const { serial } = await resolveSerial(deps.adb, void 0, config.defaultSerial, exec.signal);
|
|
2057
|
+
if (a.activity !== void 0 && a.activity !== "") {
|
|
2058
|
+
const fullActivity = a.activity.startsWith(".") ? `${a.package}${a.activity}` : a.activity;
|
|
2059
|
+
await deps.adb.shell(serial, `am start -n ${a.package}/${fullActivity}`, exec.signal);
|
|
2060
|
+
} else await deps.adb.shell(serial, `monkey -p ${a.package} -c android.intent.category.LAUNCHER 1`, exec.signal);
|
|
2061
|
+
const result = {
|
|
2062
|
+
serial,
|
|
2063
|
+
package: a.package,
|
|
2064
|
+
started: true
|
|
2065
|
+
};
|
|
2066
|
+
if (a.activity !== void 0 && a.activity !== "") result.activity = a.activity;
|
|
2067
|
+
return result;
|
|
2068
|
+
}
|
|
2069
|
+
}));
|
|
2070
|
+
ctx.tools.register(defineTool({
|
|
2071
|
+
name: "android_foreground_app",
|
|
2072
|
+
description: "Get the currently focused/foreground app and activity on the Android device. Also reports whether the screen is on. Useful for verifying which app is visible after opening an app or pressing home.",
|
|
2073
|
+
parameters: { serial: {
|
|
2074
|
+
type: "string",
|
|
2075
|
+
description: "Device serial. Omit when only one device is attached; required when multiple are connected."
|
|
2076
|
+
} },
|
|
2077
|
+
output: {
|
|
2078
|
+
schema: {
|
|
2079
|
+
type: "object",
|
|
2080
|
+
additionalProperties: false,
|
|
2081
|
+
properties: {
|
|
2082
|
+
serial: {
|
|
2083
|
+
type: "string",
|
|
2084
|
+
required: true
|
|
2085
|
+
},
|
|
2086
|
+
package: {
|
|
2087
|
+
oneOf: [{ type: "null" }, { type: "string" }],
|
|
2088
|
+
required: true
|
|
2089
|
+
},
|
|
2090
|
+
activity: {
|
|
2091
|
+
oneOf: [{ type: "null" }, { type: "string" }],
|
|
2092
|
+
required: true
|
|
2093
|
+
},
|
|
2094
|
+
window_title: {
|
|
2095
|
+
oneOf: [{ type: "null" }, { type: "string" }],
|
|
2096
|
+
required: true
|
|
2097
|
+
},
|
|
2098
|
+
screen_on: {
|
|
2099
|
+
type: "boolean",
|
|
2100
|
+
required: true
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
},
|
|
2104
|
+
render: textRender(renderForegroundApp)
|
|
2105
|
+
},
|
|
2106
|
+
async execute(args, exec) {
|
|
2107
|
+
const a = args;
|
|
2108
|
+
const config = deps.getConfig();
|
|
2109
|
+
const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
|
|
2110
|
+
const parts = (await deps.adb.shell(serial, "echo \"=FOCUS=\"; dumpsys window|grep mCurrentFocus; echo \"=WAKE=\"; dumpsys power|grep mWakefulness", exec.signal)).split(/=[A-Z]+=/);
|
|
2111
|
+
const focusOutput = parts[1] ?? "";
|
|
2112
|
+
const wakeOutput = parts[2] ?? "";
|
|
2113
|
+
const parsed = parseForegroundApp(focusOutput);
|
|
2114
|
+
const screenOn = parseWakefulness(wakeOutput);
|
|
2115
|
+
return {
|
|
2116
|
+
serial,
|
|
2117
|
+
package: parsed?.package ?? null,
|
|
2118
|
+
activity: parsed?.activity ?? null,
|
|
2119
|
+
window_title: parsed?.windowTitle ?? null,
|
|
2120
|
+
screen_on: screenOn
|
|
2121
|
+
};
|
|
2122
|
+
}
|
|
2123
|
+
}));
|
|
2124
|
+
}
|
|
2125
|
+
//#endregion
|
|
2126
|
+
//#region src/registry.ts
|
|
2127
|
+
/**
|
|
2128
|
+
* Register all 10 android_* tools into the given context.
|
|
2129
|
+
* @param ctx - the plugin context (provides `ctx.tools.register`).
|
|
2130
|
+
* @param deps - the AdbClient and live config getter.
|
|
2131
|
+
*/
|
|
2132
|
+
function registerTools(ctx, deps) {
|
|
2133
|
+
registerDeviceTools(ctx, deps);
|
|
2134
|
+
registerScreenTools(ctx, deps);
|
|
2135
|
+
registerInputTools(ctx, deps);
|
|
2136
|
+
registerAppTools(ctx, deps);
|
|
2137
|
+
}
|
|
2138
|
+
//#endregion
|
|
2139
|
+
//#region src/index.ts
|
|
2140
|
+
/**
|
|
2141
|
+
* index.ts — dsh-android-use cordis plugin entry (host half).
|
|
2142
|
+
*
|
|
2143
|
+
* 10 model-facing tools that let the AI operate an Android phone via adb:
|
|
2144
|
+
* - android_list_devices / android_device_info
|
|
2145
|
+
* - android_screenshot / android_ui_dump
|
|
2146
|
+
* - android_tap / android_swipe / android_press_key / android_input_text
|
|
2147
|
+
* - android_open_app / android_foreground_app
|
|
2148
|
+
*
|
|
2149
|
+
* Host-only (no client UI); the generic tool card is used for rendering.
|
|
2150
|
+
* Tool registration is effect-based: disposing the plugin fiber
|
|
2151
|
+
* (e.g., on config change) automatically unregisters all tools, and the
|
|
2152
|
+
* next apply() re-registers with the fresh config.
|
|
2153
|
+
*
|
|
2154
|
+
* @module @huanlin/dsh-plugin-android-use
|
|
2155
|
+
*/
|
|
2156
|
+
const name = "dsh-android-use";
|
|
2157
|
+
const inject = ["tools"];
|
|
2158
|
+
const Config = z.object({
|
|
2159
|
+
adbPath: z.string().default("adb").description("Path to the adb binary. Defaults to \"adb\" (must be on PATH)."),
|
|
2160
|
+
defaultSerial: z.string().description("Default device serial. Omit to auto-select when one device is attached; required when multiple are connected."),
|
|
2161
|
+
inputTextMode: z.union(["input", "adbkeyboard"]).default("input").description("Text input mode: \"input\" (ASCII only, uses `adb shell input text`) or \"adbkeyboard\" (Unicode, requires ADBKeyboard IME on device).")
|
|
2162
|
+
});
|
|
2163
|
+
function resolveConfig(config) {
|
|
2164
|
+
return {
|
|
2165
|
+
adbPath: typeof config.adbPath === "string" && config.adbPath !== "" ? config.adbPath : "adb",
|
|
2166
|
+
defaultSerial: typeof config.defaultSerial === "string" && config.defaultSerial !== "" ? config.defaultSerial : void 0,
|
|
2167
|
+
inputTextMode: config.inputTextMode === "adbkeyboard" ? "adbkeyboard" : "input"
|
|
2168
|
+
};
|
|
2169
|
+
}
|
|
2170
|
+
function apply(ctx, config = {}) {
|
|
2171
|
+
const resolved = resolveConfig(config);
|
|
2172
|
+
registerTools(ctx, {
|
|
2173
|
+
adb: new AdbClient(resolved.adbPath),
|
|
2174
|
+
getConfig: () => resolved
|
|
2175
|
+
});
|
|
2176
|
+
}
|
|
2177
|
+
//#endregion
|
|
2178
|
+
export { Config, apply, inject, name, resolveConfig };
|