@huanlin/dsh-plugin-android-use 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js ADDED
@@ -0,0 +1,2178 @@
1
+ import z from "schemastery";
2
+ import { spawn } from "node:child_process";
3
+ import { defineTool } from "@deepseek-ai/dsh-tools";
4
+ //#region src/adb.ts
5
+ /**
6
+ * adb.ts — spawn-based adb client with signal cancellation and binary support.
7
+ *
8
+ * Zero runtime dependencies: only `node:child_process` spawn. The class is
9
+ * injected into `registerTools` so tests substitute a fake without touching
10
+ * the real adb binary.
11
+ *
12
+ * Conventions (per plugin-development-guide.md §3):
13
+ * C5 — adb missing / device offline / spawn failure are infrastructure
14
+ * failures: they throw (the model sees a tool error, not a silent
15
+ * canonical value).
16
+ * C6 — every spawn receives `exec.signal`; aborting kills the child.
17
+ *
18
+ * @module @huanlin/dsh-plugin-android-use/src/adb
19
+ */
20
+ /**
21
+ * Parse `adb devices -l` stdout into device rows.
22
+ *
23
+ * Example input:
24
+ * ```
25
+ * List of devices attached
26
+ * 192.168.5.15:43709 device product:PJF110 model:PJF110 device:OP5CFBL1 transport_id:1
27
+ * ```
28
+ * @param output - raw stdout from `adb devices -l`.
29
+ * @returns parsed device rows (empty when no devices are attached).
30
+ */
31
+ function parseDeviceList(output) {
32
+ const lines = output.split(/\r?\n/);
33
+ const devices = [];
34
+ let started = false;
35
+ for (const line of lines) {
36
+ const trimmed = line.trim();
37
+ if (trimmed === "") continue;
38
+ if (trimmed === "List of devices attached") {
39
+ started = true;
40
+ continue;
41
+ }
42
+ if (!started) continue;
43
+ if (trimmed.startsWith("*")) continue;
44
+ const parts = trimmed.split(/\s+/);
45
+ if (parts.length < 2) continue;
46
+ const serial = parts[0];
47
+ const state = parts[1];
48
+ if (state === "device" || state === "offline" || state === "unauthorized" || state === "connecting") {
49
+ const device = {
50
+ serial,
51
+ state
52
+ };
53
+ for (let i = 2; i < parts.length; i++) {
54
+ const kv = parts[i];
55
+ const colon = kv.indexOf(":");
56
+ if (colon < 0) continue;
57
+ const key = kv.slice(0, colon);
58
+ const value = kv.slice(colon + 1);
59
+ if (key === "product") device.product = value;
60
+ else if (key === "model") device.model = value;
61
+ else if (key === "device") device.device = value;
62
+ else if (key === "transport_id") {
63
+ const id = Number(value);
64
+ if (Number.isInteger(id)) device.transportId = id;
65
+ }
66
+ }
67
+ devices.push(device);
68
+ }
69
+ }
70
+ return devices;
71
+ }
72
+ /**
73
+ * Spawn-based adb client. All methods honor `signal` for cancellation (C6);
74
+ * a non-zero exit with empty stdout throws (infrastructure failure, C5).
75
+ */
76
+ var AdbClient = class {
77
+ adbPath;
78
+ /**
79
+ * @param adbPath - path to the adb binary (default `'adb'`).
80
+ */
81
+ constructor(adbPath = "adb") {
82
+ this.adbPath = adbPath;
83
+ }
84
+ /**
85
+ * Run adb collecting stdout/stderr as UTF-8 strings.
86
+ * @param args - full argument list (e.g. `['-s', serial, 'shell', 'wm size']`).
87
+ * @param options - serial and cancellation signal.
88
+ * @returns the collected stdout, stderr, and exit code.
89
+ */
90
+ run(args, options = {}) {
91
+ return new Promise((resolve, reject) => {
92
+ const child = spawn(this.adbPath, args, {
93
+ stdio: [
94
+ "ignore",
95
+ "pipe",
96
+ "pipe"
97
+ ],
98
+ windowsHide: true
99
+ });
100
+ const stdoutChunks = [];
101
+ let stderrChunks = [];
102
+ child.stdout.on("data", (chunk) => {
103
+ stdoutChunks.push(chunk);
104
+ });
105
+ child.stderr.on("data", (chunk) => {
106
+ stderrChunks.push(chunk);
107
+ });
108
+ const onAbort = () => {
109
+ if (!child.killed) child.kill("SIGTERM");
110
+ };
111
+ const signal = options.signal;
112
+ if (signal !== void 0) {
113
+ if (signal.aborted) child.kill("SIGTERM");
114
+ signal.addEventListener("abort", onAbort, { once: true });
115
+ }
116
+ child.on("error", (error) => {
117
+ if (signal !== void 0) signal.removeEventListener("abort", onAbort);
118
+ reject(/* @__PURE__ */ new Error(`adb spawn failed: ${error.message}`));
119
+ });
120
+ child.on("close", (code) => {
121
+ if (signal !== void 0) signal.removeEventListener("abort", onAbort);
122
+ resolve({
123
+ stdout: Buffer.concat(stdoutChunks).toString("utf8"),
124
+ stderr: Buffer.concat(stderrChunks).toString("utf8"),
125
+ exitCode: code ?? -1
126
+ });
127
+ });
128
+ });
129
+ }
130
+ /**
131
+ * Run adb collecting stdout as a raw Buffer (binary-safe, for `screencap`).
132
+ * @param args - full argument list.
133
+ * @param options - serial and cancellation signal.
134
+ * @returns the raw stdout buffer.
135
+ */
136
+ runBinary(args, options = {}) {
137
+ return new Promise((resolve, reject) => {
138
+ const child = spawn(this.adbPath, args, {
139
+ stdio: [
140
+ "ignore",
141
+ "pipe",
142
+ "pipe"
143
+ ],
144
+ windowsHide: true
145
+ });
146
+ const stdoutChunks = [];
147
+ const stderrChunks = [];
148
+ child.stdout.on("data", (chunk) => {
149
+ stdoutChunks.push(chunk);
150
+ });
151
+ child.stderr.on("data", (chunk) => {
152
+ stderrChunks.push(chunk);
153
+ });
154
+ const onAbort = () => {
155
+ if (!child.killed) child.kill("SIGTERM");
156
+ };
157
+ const signal = options.signal;
158
+ if (signal !== void 0) {
159
+ if (signal.aborted) child.kill("SIGTERM");
160
+ signal.addEventListener("abort", onAbort, { once: true });
161
+ }
162
+ child.on("error", (error) => {
163
+ if (signal !== void 0) signal.removeEventListener("abort", onAbort);
164
+ reject(/* @__PURE__ */ new Error(`adb spawn failed: ${error.message}`));
165
+ });
166
+ child.on("close", (code) => {
167
+ if (signal !== void 0) signal.removeEventListener("abort", onAbort);
168
+ const stdout = Buffer.concat(stdoutChunks);
169
+ const stderr = Buffer.concat(stderrChunks).toString("utf8");
170
+ if (code !== 0) {
171
+ reject(/* @__PURE__ */ new Error(`adb exited with code ${code}: ${stderr || "(no stderr)"}`));
172
+ return;
173
+ }
174
+ if (signal?.aborted) {
175
+ reject(/* @__PURE__ */ new Error("adb was cancelled"));
176
+ return;
177
+ }
178
+ resolve(stdout);
179
+ });
180
+ });
181
+ }
182
+ /**
183
+ * List attached devices via `adb devices -l`.
184
+ * @returns parsed device rows.
185
+ */
186
+ async devices(signal) {
187
+ return parseDeviceList((await this.run(["devices", "-l"], { signal })).stdout);
188
+ }
189
+ /**
190
+ * Run a shell command on the device (`adb -s <serial> shell <cmd>`).
191
+ * @returns the command stdout as a string.
192
+ */
193
+ async shell(serial, command, signal) {
194
+ const result = await this.run([
195
+ "-s",
196
+ serial,
197
+ "shell",
198
+ command
199
+ ], {
200
+ serial,
201
+ signal
202
+ });
203
+ if (result.exitCode !== 0) throw new Error(`adb shell failed (exit ${result.exitCode}): ${result.stderr || result.stdout}`);
204
+ return result.stdout;
205
+ }
206
+ /**
207
+ * Run an exec-out command (`adb -s <serial> exec-out <cmd>`), binary-safe.
208
+ * @returns the command stdout as a string (UTF-8 decoded).
209
+ */
210
+ async execOut(serial, command, signal) {
211
+ const result = await this.run([
212
+ "-s",
213
+ serial,
214
+ "exec-out",
215
+ command
216
+ ], {
217
+ serial,
218
+ signal
219
+ });
220
+ if (result.exitCode !== 0) throw new Error(`adb exec-out failed (exit ${result.exitCode}): ${result.stderr}`);
221
+ return result.stdout;
222
+ }
223
+ /**
224
+ * Run an exec-out command collecting raw bytes (for `screencap -p`).
225
+ * @returns the command stdout as a Buffer.
226
+ */
227
+ async execOutBinary(serial, command, signal) {
228
+ return this.runBinary([
229
+ "-s",
230
+ serial,
231
+ "exec-out",
232
+ command
233
+ ], {
234
+ serial,
235
+ signal
236
+ });
237
+ }
238
+ };
239
+ /** Parse `wm size` output into `{ width, height }`. */
240
+ function parseWmSize$1(output) {
241
+ const overrideMatch = /Override size:\s*(\d+)x(\d+)/.exec(output);
242
+ if (overrideMatch !== null) return {
243
+ width: Number(overrideMatch[1]),
244
+ height: Number(overrideMatch[2])
245
+ };
246
+ const physicalMatch = /Physical size:\s*(\d+)x(\d+)/.exec(output);
247
+ if (physicalMatch !== null) return {
248
+ width: Number(physicalMatch[1]),
249
+ height: Number(physicalMatch[2])
250
+ };
251
+ throw new Error(`could not parse "wm size" output: ${output.trim()}`);
252
+ }
253
+ /**
254
+ * Resolve the target device serial following the priority order:
255
+ * 1. `args.serial` (explicit per-call parameter)
256
+ * 2. `config.defaultSerial` (plugin config)
257
+ * 3. single attached device (auto-select)
258
+ * 4. multiple devices → throw "multiple devices, pass serial"
259
+ *
260
+ * @param adb - the adb client (used to list devices when needed).
261
+ * @param argSerial - the per-call `serial` argument, if the model supplied one.
262
+ * @param configSerial - the plugin config `defaultSerial`, if configured.
263
+ * @param signal - cancellation for the device-list query.
264
+ * @returns the resolved serial and its source.
265
+ */
266
+ async function resolveSerial(adb, argSerial, configSerial, signal) {
267
+ if (argSerial !== void 0 && argSerial !== "") return {
268
+ serial: argSerial,
269
+ source: "param"
270
+ };
271
+ if (configSerial !== void 0 && configSerial !== "") return {
272
+ serial: configSerial,
273
+ source: "config"
274
+ };
275
+ const ready = (await adb.devices(signal)).filter((d) => d.state === "device");
276
+ if (ready.length === 0) throw new Error("no device is ready; connect a device or specify a serial");
277
+ if (ready.length > 1) {
278
+ const list = ready.map((d) => d.serial).join(", ");
279
+ throw new Error(`multiple devices attached (${list}); pass the "serial" parameter to select one`);
280
+ }
281
+ return {
282
+ serial: ready[0].serial,
283
+ source: "auto"
284
+ };
285
+ }
286
+ //#endregion
287
+ //#region src/tools/device.ts
288
+ /** Parse `wm size` output, preferring the override (effective) resolution. */
289
+ function parseWmSize(output) {
290
+ const physical = /Physical size:\s*(\d+)x(\d+)/.exec(output);
291
+ const override = /Override size:\s*(\d+)x(\d+)/.exec(output);
292
+ const pw = physical !== null ? Number(physical[1]) : 0;
293
+ const ph = physical !== null ? Number(physical[2]) : 0;
294
+ return {
295
+ width: override !== null ? Number(override[1]) : pw,
296
+ height: override !== null ? Number(override[2]) : ph,
297
+ physicalWidth: pw,
298
+ physicalHeight: ph
299
+ };
300
+ }
301
+ /** Parse `wm density` output, preferring the override density. */
302
+ function parseWmDensity(output) {
303
+ const physical = /Physical density:\s*(\d+)/.exec(output);
304
+ const override = /Override density:\s*(\d+)/.exec(output);
305
+ const pd = physical !== null ? Number(physical[1]) : 0;
306
+ return {
307
+ density: override !== null ? Number(override[1]) : pd,
308
+ physicalDensity: pd
309
+ };
310
+ }
311
+ /** Parse `dumpsys power` output for the wakefulness state. */
312
+ function parseWakefulness$1(output) {
313
+ return /mWakefulness=Awake/.test(output);
314
+ }
315
+ /** Convert an `AdbDevice` to the canonical list-devices entry. */
316
+ function deviceToEntry(d) {
317
+ const entry = {
318
+ serial: d.serial,
319
+ state: d.state
320
+ };
321
+ if (d.product !== void 0) entry.product = d.product;
322
+ if (d.model !== void 0) entry.model = d.model;
323
+ if (d.device !== void 0) entry.device = d.device;
324
+ return entry;
325
+ }
326
+ function renderListDevices(value) {
327
+ if (value.devices.length === 0) return "No devices attached.";
328
+ const lines = [`Found ${value.devices.length} device(s):`];
329
+ for (const d of value.devices) {
330
+ const parts = [`state: ${d.state}`];
331
+ if (d.model !== void 0) parts.push(`model: ${d.model}`);
332
+ if (d.product !== void 0) parts.push(`product: ${d.product}`);
333
+ lines.push(` - ${d.serial} (${parts.join(", ")})`);
334
+ }
335
+ return lines.join("\n");
336
+ }
337
+ function renderDeviceInfo(value) {
338
+ return [
339
+ `Device: ${value.serial}`,
340
+ ` Model: ${value.model} (${value.brand})`,
341
+ ` Android: ${value.android_version} (SDK ${value.sdk})`,
342
+ ` Screen: ${value.screen.width}x${value.screen.height} @ ${value.screen.density}dpi`,
343
+ ` Screen on: ${value.screen_on}`
344
+ ].join("\n");
345
+ }
346
+ function textRender$2(fn) {
347
+ return (_args, value) => [{
348
+ type: "text",
349
+ text: fn(value)
350
+ }];
351
+ }
352
+ /** Register `android_list_devices` and `android_device_info`. */
353
+ function registerDeviceTools(ctx, deps) {
354
+ ctx.tools.register(defineTool({
355
+ name: "android_list_devices",
356
+ description: "List all Android devices attached via adb. Returns each device's serial, state, product, model, and device name. Call this first to discover available devices before using other android_* tools.",
357
+ parameters: {},
358
+ output: {
359
+ schema: {
360
+ type: "object",
361
+ additionalProperties: false,
362
+ properties: {
363
+ devices: {
364
+ type: "array",
365
+ items: {
366
+ type: "object",
367
+ additionalProperties: false,
368
+ properties: {
369
+ serial: {
370
+ type: "string",
371
+ required: true
372
+ },
373
+ state: {
374
+ type: "string",
375
+ required: true
376
+ },
377
+ product: { type: "string" },
378
+ model: { type: "string" },
379
+ device: { type: "string" }
380
+ }
381
+ }
382
+ },
383
+ total: {
384
+ type: "integer",
385
+ required: true
386
+ }
387
+ }
388
+ },
389
+ render: textRender$2(renderListDevices)
390
+ },
391
+ async execute(_args, exec) {
392
+ const devices = await deps.adb.devices(exec.signal);
393
+ return {
394
+ devices: devices.map(deviceToEntry),
395
+ total: devices.length
396
+ };
397
+ }
398
+ }));
399
+ ctx.tools.register(defineTool({
400
+ name: "android_device_info",
401
+ description: "Get detailed information about an Android device: model, brand, Android version, SDK level, screen resolution, density, and whether the screen is currently on. Pass \"serial\" to target a specific device, or omit it when only one device is attached.",
402
+ parameters: { serial: {
403
+ type: "string",
404
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
405
+ } },
406
+ output: {
407
+ schema: {
408
+ type: "object",
409
+ additionalProperties: false,
410
+ properties: {
411
+ serial: {
412
+ type: "string",
413
+ required: true
414
+ },
415
+ model: {
416
+ type: "string",
417
+ required: true
418
+ },
419
+ brand: {
420
+ type: "string",
421
+ required: true
422
+ },
423
+ android_version: {
424
+ type: "string",
425
+ required: true
426
+ },
427
+ sdk: {
428
+ type: "integer",
429
+ required: true
430
+ },
431
+ screen: {
432
+ type: "object",
433
+ additionalProperties: false,
434
+ required: true,
435
+ properties: {
436
+ width: {
437
+ type: "integer",
438
+ required: true
439
+ },
440
+ height: {
441
+ type: "integer",
442
+ required: true
443
+ },
444
+ density: {
445
+ type: "integer",
446
+ required: true
447
+ }
448
+ }
449
+ },
450
+ screen_on: {
451
+ type: "boolean",
452
+ required: true
453
+ }
454
+ }
455
+ },
456
+ render: textRender$2(renderDeviceInfo)
457
+ },
458
+ async execute(args, exec) {
459
+ const a = args;
460
+ const config = deps.getConfig();
461
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
462
+ const sections = splitByMarkers(await deps.adb.shell(serial, "echo \"=SZ=\"; wm size; echo \"=DN=\"; wm density; echo \"=MD=\"; getprop ro.product.model; echo \"=BR=\"; getprop ro.product.brand; echo \"=VR=\"; getprop ro.build.version.release; echo \"=SDK=\"; getprop ro.build.version.sdk; echo \"=WK=\"; dumpsys power|grep mWakefulness", exec.signal));
463
+ const sz = parseWmSize(sections.sz);
464
+ const dn = parseWmDensity(sections.dn);
465
+ return {
466
+ serial,
467
+ model: sections.md.trim(),
468
+ brand: sections.br.trim(),
469
+ android_version: sections.vr.trim(),
470
+ sdk: Number(sections.sdk.trim()) || 0,
471
+ screen: {
472
+ width: sz.width,
473
+ height: sz.height,
474
+ density: dn.density
475
+ },
476
+ screen_on: parseWakefulness$1(sections.wk)
477
+ };
478
+ }
479
+ }));
480
+ }
481
+ /** Split a batched shell output by `=XX=` markers into named sections. */
482
+ function splitByMarkers(output) {
483
+ const parts = output.split(/=[A-Z]+=/);
484
+ return {
485
+ sz: parts[1] ?? "",
486
+ dn: parts[2] ?? "",
487
+ md: parts[3] ?? "",
488
+ br: parts[4] ?? "",
489
+ vr: parts[5] ?? "",
490
+ sdk: parts[6] ?? "",
491
+ wk: parts[7] ?? ""
492
+ };
493
+ }
494
+ //#endregion
495
+ //#region src/route.ts
496
+ /**
497
+ * Best-effort check whether the current model route accepts image input.
498
+ * Returns false when the llm service is absent, the route cannot be resolved,
499
+ * or the resolved model does not declare image input. Never throws.
500
+ */
501
+ async function routeIsImageCapable(ctx, exec, signal) {
502
+ const llm = ctx.get("llm");
503
+ if (llm === void 0) return false;
504
+ const agent = exec.agent;
505
+ if (agent === void 0) return false;
506
+ const routed = agent.session.requestHeader()?.config;
507
+ const provider = routed?.provider ?? agent.options.provider;
508
+ const model = routed?.model ?? agent.options.model;
509
+ if (provider === void 0 || model === void 0) return false;
510
+ try {
511
+ return (await llm.resolveModelInfo(provider, model, signal)).inputModalities?.includes("image") === true;
512
+ } catch {
513
+ return false;
514
+ }
515
+ }
516
+ //#endregion
517
+ //#region src/xml.ts
518
+ /**
519
+ * Parse a `[left,top][right,bottom]` bounds string.
520
+ * @param raw - the attribute value, e.g. `"[0,0][1080,2414]"`.
521
+ * @returns the numeric bounds, or null when the string does not match.
522
+ */
523
+ function parseBounds(raw) {
524
+ const match = /^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/.exec(raw);
525
+ if (match === null) return null;
526
+ return {
527
+ left: Number(match[1]),
528
+ top: Number(match[2]),
529
+ right: Number(match[3]),
530
+ bottom: Number(match[4])
531
+ };
532
+ }
533
+ /** Compute the center of a bounds rect. */
534
+ function centerOfBounds(bounds) {
535
+ return {
536
+ x: Math.floor((bounds.left + bounds.right) / 2),
537
+ y: Math.floor((bounds.top + bounds.bottom) / 2)
538
+ };
539
+ }
540
+ /** Decode the five XML entity references uiautomator emits. */
541
+ function decodeEntities(value) {
542
+ return value.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, "\"").replace(/&apos;/g, "'");
543
+ }
544
+ /**
545
+ * Extract one attribute value from a `<node ...>` tag string.
546
+ * @param tag - the raw `<node ...` text up to the closing `>` or `/>`.
547
+ * @param name - the attribute name without quotes.
548
+ * @returns the decoded value, or empty string when absent.
549
+ */
550
+ function attrValue(tag, name) {
551
+ const match = new RegExp(`${name}="((?:[^"&]|&(?:amp|lt|gt|quot|apos);)*)"`, "u").exec(tag);
552
+ if (match === null) return "";
553
+ return decodeEntities(match[1]);
554
+ }
555
+ /** Parse a boolean attribute: `"true"` → true, everything else → false. */
556
+ function boolAttr(tag, name) {
557
+ return attrValue(tag, name) === "true";
558
+ }
559
+ /**
560
+ * Scan the XML for all `<node>` opening, self-closing, and closing tags in
561
+ * document order. Closing tags are tracked so depth stays accurate.
562
+ */
563
+ const TAG_PATTERN = /<(\/?)node\b([^>]*?)(\/?)>/gu;
564
+ function scanTags(xml) {
565
+ const tags = [];
566
+ let match;
567
+ const pattern = new RegExp(TAG_PATTERN);
568
+ while ((match = pattern.exec(xml)) !== null) {
569
+ const closing = match[1] === "/";
570
+ const selfClosing = !closing && match[3] === "/";
571
+ tags.push({
572
+ raw: match[0],
573
+ closing,
574
+ selfClosing
575
+ });
576
+ }
577
+ return tags;
578
+ }
579
+ /**
580
+ * Parse a uiautomator dump XML into a flat node list with screen metadata.
581
+ *
582
+ * The root `<hierarchy rotation="N">` element carries the rotation; the
583
+ * screen dimensions are read from the first (root) node's bounds, which
584
+ * uiautomator always emits as the full display rectangle.
585
+ *
586
+ * Closing `</node>` tags are tracked so `depth` accurately reflects the tree
587
+ * nesting level (root node = depth 0, its children = depth 1, etc.).
588
+ *
589
+ * @param xml - the raw XML string from `uiautomator dump` + `cat`.
590
+ * @returns the parsed dump. Empty/malformed XML yields zero nodes and
591
+ * zero dimensions rather than throwing (the model still gets a result).
592
+ */
593
+ function parseUiDumpXml(xml) {
594
+ const rotationMatch = /rotation="(-?\d+)"/.exec(xml);
595
+ const rotation = rotationMatch !== null ? Number(rotationMatch[1]) : 0;
596
+ const tags = scanTags(xml);
597
+ const nodes = [];
598
+ let depth = 0;
599
+ let screenWidth = 0;
600
+ let screenHeight = 0;
601
+ let isFirstNode = true;
602
+ for (const tag of tags) {
603
+ if (tag.closing) {
604
+ if (depth > 0) depth -= 1;
605
+ continue;
606
+ }
607
+ const raw = tag.raw;
608
+ const boundsRaw = attrValue(raw, "bounds");
609
+ const bounds = boundsRaw !== "" ? parseBounds(boundsRaw) : null;
610
+ if (isFirstNode && bounds !== null) {
611
+ screenWidth = bounds.right;
612
+ screenHeight = bounds.bottom;
613
+ isFirstNode = false;
614
+ }
615
+ const node = {
616
+ text: attrValue(raw, "text"),
617
+ content_desc: attrValue(raw, "content-desc"),
618
+ resource_id: attrValue(raw, "resource-id"),
619
+ class: attrValue(raw, "class"),
620
+ package: attrValue(raw, "package"),
621
+ bounds,
622
+ center: bounds !== null ? centerOfBounds(bounds) : null,
623
+ clickable: boolAttr(raw, "clickable"),
624
+ long_clickable: boolAttr(raw, "long-clickable"),
625
+ focusable: boolAttr(raw, "focusable"),
626
+ scrollable: boolAttr(raw, "scrollable"),
627
+ enabled: boolAttr(raw, "enabled"),
628
+ password: boolAttr(raw, "password"),
629
+ selected: boolAttr(raw, "selected"),
630
+ checked: boolAttr(raw, "checked"),
631
+ depth
632
+ };
633
+ nodes.push(node);
634
+ if (!tag.selfClosing) depth += 1;
635
+ }
636
+ return {
637
+ rotation,
638
+ screen_width: screenWidth,
639
+ screen_height: screenHeight,
640
+ nodes
641
+ };
642
+ }
643
+ //#endregion
644
+ //#region src/image.ts
645
+ /**
646
+ * image.ts — fit a PNG within the attachment store's per-side pixel limit.
647
+ *
648
+ * The attachment store enforces `maxImageDimension` (2000px by default).
649
+ * This module downscales only when necessary, returning a scale factor so
650
+ * callers can map device-native coordinates to image space for annotation.
651
+ *
652
+ * @module @huanlin/dsh-plugin-android-use/src/image
653
+ */
654
+ /**
655
+ * Compute the uniform scale factor applied when a screenshot of the given
656
+ * device resolution is fit within `maxDimension`.
657
+ *
658
+ * The scale is uniform (same for X and Y) because `fitToMaxDimension` uses
659
+ * `fit: 'inside'` which preserves aspect ratio.
660
+ *
661
+ * @returns a value in (0, 1]. Returns 1 when no scaling is needed.
662
+ */
663
+ function computeScale(deviceWidth, deviceHeight, maxDimension) {
664
+ if (deviceWidth <= 0 || deviceHeight <= 0) return 1;
665
+ return Math.min(1, maxDimension / Math.max(deviceWidth, deviceHeight));
666
+ }
667
+ /**
668
+ * Downscale a PNG so its longest side fits within `maxDimension`, preserving
669
+ * aspect ratio. Returns the original bytes unchanged when already within the
670
+ * limit. Uses `sharp` for the resize.
671
+ *
672
+ * @param data - the raw PNG bytes from `screencap -p`.
673
+ * @param maxDimension - the attachment store's per-side pixel limit.
674
+ */
675
+ async function fitToMaxDimension(data, maxDimension) {
676
+ const sharp = (await import("sharp")).default;
677
+ let meta;
678
+ try {
679
+ meta = await sharp(data).metadata();
680
+ } catch {
681
+ throw new Error("screencap returned an image with no dimensions");
682
+ }
683
+ const origW = meta.width ?? 0;
684
+ const origH = meta.height ?? 0;
685
+ if (origW === 0 || origH === 0) throw new Error("screencap returned an image with no dimensions");
686
+ if (Math.max(origW, origH) <= maxDimension) return {
687
+ data,
688
+ width: origW,
689
+ height: origH,
690
+ scaleX: 1,
691
+ scaleY: 1
692
+ };
693
+ const result = await sharp(data, {
694
+ failOn: "error",
695
+ limitInputPixels: false
696
+ }).resize({
697
+ width: maxDimension,
698
+ height: maxDimension,
699
+ fit: "inside",
700
+ withoutEnlargement: true
701
+ }).png().toBuffer({ resolveWithObject: true });
702
+ return {
703
+ data: new Uint8Array(result.data),
704
+ width: result.info.width,
705
+ height: result.info.height,
706
+ scaleX: result.info.width / origW,
707
+ scaleY: result.info.height / origH
708
+ };
709
+ }
710
+ //#endregion
711
+ //#region src/tools/screen.ts
712
+ function renderScreenshot(_args, value) {
713
+ const v = value;
714
+ const text = [
715
+ `Screenshot captured: ${v.width}x${v.height} px, ${v.bytes} bytes (image/png).`,
716
+ `Device resolution: ${v.device_width}x${v.device_height}, scale: ${v.scale.toFixed(4)}.`,
717
+ `Coordinates from android_ui_dump and android_tap use the ${v.width}x${v.height} image space.`,
718
+ `Image emitted to model context: ${v.image_emitted ? "yes" : "no"}${v.image_emitted ? "" : " (current route is not image-capable; use android_ui_dump for screen perception)"}.`
719
+ ].join("\n");
720
+ if (v.image_emitted) {
721
+ const ref = {
722
+ attachmentId: v.image.attachmentId,
723
+ mediaType: "image/png",
724
+ bytes: v.image.bytes,
725
+ width: v.image.width,
726
+ height: v.image.height
727
+ };
728
+ return [{
729
+ type: "text",
730
+ text
731
+ }, {
732
+ type: "image",
733
+ attachment: ref
734
+ }];
735
+ }
736
+ return [{
737
+ type: "text",
738
+ text
739
+ }];
740
+ }
741
+ function nodeToTextLine(node, index) {
742
+ const parts = [`[${index}]`];
743
+ if (node.text !== "") parts.push(`text=${JSON.stringify(node.text)}`);
744
+ if (node.content_desc !== "") parts.push(`desc=${JSON.stringify(node.content_desc)}`);
745
+ if (node.resource_id !== "") parts.push(`id=${node.resource_id}`);
746
+ parts.push(`class=${node.class}`);
747
+ if (node.bounds !== null) parts.push(`bounds=[${node.bounds.left},${node.bounds.top}][${node.bounds.right},${node.bounds.bottom}]`);
748
+ if (node.center !== null) parts.push(`center=(${node.center.x},${node.center.y})`);
749
+ const flags = [];
750
+ if (node.clickable) flags.push("clickable");
751
+ if (node.long_clickable) flags.push("long-clickable");
752
+ if (node.scrollable) flags.push("scrollable");
753
+ if (node.focusable) flags.push("focusable");
754
+ if (!node.enabled) flags.push("disabled");
755
+ if (node.checked) flags.push("checked");
756
+ if (node.password) flags.push("password");
757
+ if (node.selected) flags.push("selected");
758
+ if (flags.length > 0) parts.push(flags.join(","));
759
+ return ` ${parts.join(" | ")}`;
760
+ }
761
+ function scaleDump(dump, scale) {
762
+ if (scale >= 1) return dump;
763
+ const r = (n) => Math.round(n * scale);
764
+ const nodes = dump.nodes.map((node) => ({
765
+ ...node,
766
+ bounds: node.bounds !== null ? {
767
+ left: r(node.bounds.left),
768
+ top: r(node.bounds.top),
769
+ right: r(node.bounds.right),
770
+ bottom: r(node.bounds.bottom)
771
+ } : null,
772
+ center: node.center !== null ? {
773
+ x: r(node.center.x),
774
+ y: r(node.center.y)
775
+ } : null
776
+ }));
777
+ return {
778
+ rotation: dump.rotation,
779
+ screen_width: dump.screen_width,
780
+ screen_height: dump.screen_height,
781
+ nodes
782
+ };
783
+ }
784
+ function renderUiDump(_args, value) {
785
+ const v = value;
786
+ const total = v.nodes.length;
787
+ const interactive = v.nodes.filter((n) => n.text !== "" || n.content_desc !== "" || n.clickable || n.long_clickable || n.scrollable || n.focusable);
788
+ const lines = [
789
+ `UI dump: ${total} nodes total (${interactive.length} interactive/text), screen ${v.screen_width}x${v.screen_height}, rotation ${v.rotation}.`,
790
+ `Coordinates are in screenshot image space (${v.image_width}x${v.image_height}, scale ${v.scale.toFixed(4)}). Use these center values directly with android_tap.`,
791
+ `Showing ${interactive.length} interactive/text nodes:`
792
+ ];
793
+ let idx = 0;
794
+ for (const node of v.nodes) {
795
+ if (!(node.text !== "" || node.content_desc !== "" || node.clickable || node.long_clickable || node.scrollable || node.focusable)) continue;
796
+ lines.push(nodeToTextLine(node, idx));
797
+ idx++;
798
+ }
799
+ return [{
800
+ type: "text",
801
+ text: lines.join("\n")
802
+ }];
803
+ }
804
+ /** Register `android_screenshot` and `android_ui_dump`. */
805
+ function registerScreenTools(ctx, deps) {
806
+ ctx.tools.register(defineTool({
807
+ name: "android_screenshot",
808
+ description: "Capture a screenshot from the Android device. The image may be scaled to fit the attachment store pixel limit; the result includes device_width, device_height, and scale so you know the mapping. Coordinates from android_ui_dump and android_tap use the scaled image space. The image is always saved to the attachment store (visible in the UI card). The image is emitted to the model only when the current model route accepts image input; otherwise, use android_ui_dump for text-based screen perception. Pass \"serial\" to target a specific device.",
809
+ parameters: { serial: {
810
+ type: "string",
811
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
812
+ } },
813
+ output: {
814
+ schema: {
815
+ type: "object",
816
+ additionalProperties: false,
817
+ properties: {
818
+ serial: {
819
+ type: "string",
820
+ required: true
821
+ },
822
+ width: {
823
+ type: "integer",
824
+ required: true
825
+ },
826
+ height: {
827
+ type: "integer",
828
+ required: true
829
+ },
830
+ bytes: {
831
+ type: "integer",
832
+ required: true
833
+ },
834
+ device_width: {
835
+ type: "integer",
836
+ required: true
837
+ },
838
+ device_height: {
839
+ type: "integer",
840
+ required: true
841
+ },
842
+ scale: {
843
+ type: "number",
844
+ required: true
845
+ },
846
+ image: {
847
+ type: "object",
848
+ additionalProperties: false,
849
+ required: true,
850
+ properties: {
851
+ attachmentId: {
852
+ type: "string",
853
+ required: true
854
+ },
855
+ mediaType: {
856
+ type: "string",
857
+ const: "image/png",
858
+ required: true
859
+ },
860
+ bytes: {
861
+ type: "integer",
862
+ required: true
863
+ },
864
+ width: {
865
+ type: "integer",
866
+ required: true
867
+ },
868
+ height: {
869
+ type: "integer",
870
+ required: true
871
+ }
872
+ }
873
+ },
874
+ image_emitted: {
875
+ type: "boolean",
876
+ required: true
877
+ }
878
+ }
879
+ },
880
+ render: renderScreenshot
881
+ },
882
+ async execute(args, exec) {
883
+ const a = args;
884
+ const config = deps.getConfig();
885
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
886
+ const attachments = ctx.get("attachments");
887
+ if (attachments === void 0) throw new Error("cannot capture screenshot: no attachment service is mounted");
888
+ const raw = await deps.adb.execOutBinary(serial, "screencap -p", exec.signal);
889
+ const fitted = await fitToMaxDimension(new Uint8Array(raw), attachments.imageLimits.maxImageDimension);
890
+ const ref = await attachments.saveImage({
891
+ data: fitted.data,
892
+ mediaType: "image/png",
893
+ name: "screenshot.png"
894
+ });
895
+ const imageEmitted = await routeIsImageCapable(ctx, exec, exec.signal);
896
+ const scale = fitted.scaleX;
897
+ return {
898
+ serial,
899
+ width: ref.width,
900
+ height: ref.height,
901
+ bytes: ref.bytes,
902
+ device_width: Math.round(ref.width / fitted.scaleX),
903
+ device_height: Math.round(ref.height / fitted.scaleY),
904
+ scale,
905
+ image: {
906
+ attachmentId: ref.attachmentId,
907
+ mediaType: "image/png",
908
+ bytes: ref.bytes,
909
+ width: ref.width,
910
+ height: ref.height
911
+ },
912
+ image_emitted: imageEmitted
913
+ };
914
+ }
915
+ }));
916
+ ctx.tools.register(defineTool({
917
+ name: "android_ui_dump",
918
+ description: "Dump the Android accessibility tree (UI hierarchy) as a structured node list. Each node includes text, content description, resource-id, class, bounds, center coordinates, and interaction flags (clickable, scrollable, etc.). All coordinates are in screenshot image space (scaled to match the screenshot from android_screenshot). Use node \"center\" values directly with android_tap. Pass \"serial\" to target a specific device.",
919
+ parameters: { serial: {
920
+ type: "string",
921
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
922
+ } },
923
+ output: {
924
+ schema: {
925
+ type: "object",
926
+ additionalProperties: false,
927
+ properties: {
928
+ serial: {
929
+ type: "string",
930
+ required: true
931
+ },
932
+ screen_width: {
933
+ type: "integer",
934
+ required: true
935
+ },
936
+ screen_height: {
937
+ type: "integer",
938
+ required: true
939
+ },
940
+ image_width: {
941
+ type: "integer",
942
+ required: true
943
+ },
944
+ image_height: {
945
+ type: "integer",
946
+ required: true
947
+ },
948
+ scale: {
949
+ type: "number",
950
+ required: true
951
+ },
952
+ rotation: {
953
+ type: "integer",
954
+ required: true
955
+ },
956
+ nodes: {
957
+ type: "array",
958
+ items: {
959
+ type: "object",
960
+ additionalProperties: false,
961
+ properties: {
962
+ text: {
963
+ type: "string",
964
+ required: true
965
+ },
966
+ content_desc: {
967
+ type: "string",
968
+ required: true
969
+ },
970
+ resource_id: {
971
+ type: "string",
972
+ required: true
973
+ },
974
+ class: {
975
+ type: "string",
976
+ required: true
977
+ },
978
+ package: {
979
+ type: "string",
980
+ required: true
981
+ },
982
+ bounds: {
983
+ oneOf: [{ type: "null" }, {
984
+ type: "object",
985
+ additionalProperties: false,
986
+ properties: {
987
+ left: {
988
+ type: "integer",
989
+ required: true
990
+ },
991
+ top: {
992
+ type: "integer",
993
+ required: true
994
+ },
995
+ right: {
996
+ type: "integer",
997
+ required: true
998
+ },
999
+ bottom: {
1000
+ type: "integer",
1001
+ required: true
1002
+ }
1003
+ }
1004
+ }],
1005
+ required: true
1006
+ },
1007
+ center: {
1008
+ oneOf: [{ type: "null" }, {
1009
+ type: "object",
1010
+ additionalProperties: false,
1011
+ properties: {
1012
+ x: {
1013
+ type: "integer",
1014
+ required: true
1015
+ },
1016
+ y: {
1017
+ type: "integer",
1018
+ required: true
1019
+ }
1020
+ }
1021
+ }],
1022
+ required: true
1023
+ },
1024
+ clickable: {
1025
+ type: "boolean",
1026
+ required: true
1027
+ },
1028
+ long_clickable: {
1029
+ type: "boolean",
1030
+ required: true
1031
+ },
1032
+ focusable: {
1033
+ type: "boolean",
1034
+ required: true
1035
+ },
1036
+ scrollable: {
1037
+ type: "boolean",
1038
+ required: true
1039
+ },
1040
+ enabled: {
1041
+ type: "boolean",
1042
+ required: true
1043
+ },
1044
+ password: {
1045
+ type: "boolean",
1046
+ required: true
1047
+ },
1048
+ selected: {
1049
+ type: "boolean",
1050
+ required: true
1051
+ },
1052
+ checked: {
1053
+ type: "boolean",
1054
+ required: true
1055
+ },
1056
+ depth: {
1057
+ type: "integer",
1058
+ required: true
1059
+ }
1060
+ }
1061
+ }
1062
+ }
1063
+ }
1064
+ },
1065
+ render: renderUiDump
1066
+ },
1067
+ async execute(args, exec) {
1068
+ const a = args;
1069
+ const config = deps.getConfig();
1070
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
1071
+ const dumpPath = `/sdcard/dsh_ui_dump_${Date.now()}.xml`;
1072
+ await deps.adb.shell(serial, `uiautomator dump ${dumpPath}`, exec.signal);
1073
+ const dump = parseUiDumpXml(await deps.adb.execOut(serial, `cat ${dumpPath}`, exec.signal));
1074
+ const maxDim = ctx.get("attachments")?.imageLimits.maxImageDimension ?? Infinity;
1075
+ const scale = computeScale(dump.screen_width, dump.screen_height, maxDim);
1076
+ const scaled = scaleDump(dump, scale);
1077
+ return {
1078
+ serial,
1079
+ screen_width: dump.screen_width,
1080
+ screen_height: dump.screen_height,
1081
+ image_width: Math.round(dump.screen_width * scale),
1082
+ image_height: Math.round(dump.screen_height * scale),
1083
+ scale,
1084
+ rotation: dump.rotation,
1085
+ nodes: scaled.nodes
1086
+ };
1087
+ }
1088
+ }));
1089
+ }
1090
+ //#endregion
1091
+ //#region src/annotate.ts
1092
+ /**
1093
+ * Overlay a prominent tap marker on a PNG at device-native coordinates.
1094
+ *
1095
+ * The marker is a layered target: outer glow zone, white-bordered red ring,
1096
+ * solid center dot, and long crosshair arms — designed to be visible on any
1097
+ * background.
1098
+ *
1099
+ * @param png - the original PNG bytes (no scaling).
1100
+ * @param x - tap X in device pixels.
1101
+ * @param y - tap Y in device pixels.
1102
+ * @returns the annotated PNG bytes and dimensions.
1103
+ */
1104
+ async function annotateTap(png, x, y) {
1105
+ const sharp = (await import("sharp")).default;
1106
+ let meta;
1107
+ try {
1108
+ meta = await sharp(png).metadata();
1109
+ } catch {
1110
+ throw new Error("cannot annotate: screenshot has no dimensions");
1111
+ }
1112
+ const width = meta.width ?? 0;
1113
+ const height = meta.height ?? 0;
1114
+ if (width === 0 || height === 0) throw new Error("cannot annotate: screenshot has no dimensions");
1115
+ const r = Math.max(28, Math.round(Math.min(width, height) * .04));
1116
+ const ringW = Math.max(4, Math.round(r * .14));
1117
+ const lineW = Math.max(3, Math.round(r * .1));
1118
+ const armLen = r * 2.4;
1119
+ const red = "#FF1744";
1120
+ const white = "#FFFFFF";
1121
+ const svg = `<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg"><circle cx="${x}" cy="${y}" r="${Math.round(r * 2.2)}" fill="${red}" opacity="0.08"/><circle cx="${x}" cy="${y}" r="${r}" fill="none" stroke="${white}" stroke-width="${ringW + 4}" opacity="0.45"/><circle cx="${x}" cy="${y}" r="${r}" fill="none" stroke="${red}" stroke-width="${ringW}" opacity="0.9"/><circle cx="${x}" cy="${y}" r="${Math.round(r * .82)}" fill="none" stroke="${white}" stroke-width="2" opacity="0.5"/><line x1="${Math.round(x - armLen)}" y1="${y}" x2="${Math.round(x + armLen)}" y2="${y}" stroke="${white}" stroke-width="${lineW + 3}" opacity="0.35"/><line x1="${x}" y1="${Math.round(y - armLen)}" x2="${x}" y2="${Math.round(y + armLen)}" stroke="${white}" stroke-width="${lineW + 3}" opacity="0.35"/><line x1="${Math.round(x - armLen)}" y1="${y}" x2="${Math.round(x + armLen)}" y2="${y}" stroke="${red}" stroke-width="${lineW}" opacity="0.8"/><line x1="${x}" y1="${Math.round(y - armLen)}" x2="${x}" y2="${Math.round(y + armLen)}" stroke="${red}" stroke-width="${lineW}" opacity="0.8"/><circle cx="${x}" cy="${y}" r="${Math.round(r * .24)}" fill="none" stroke="${white}" stroke-width="2" opacity="0.6"/><circle cx="${x}" cy="${y}" r="${Math.round(r * .24)}" fill="${red}" opacity="0.85"/></svg>`;
1122
+ const buf = await sharp(png).composite([{
1123
+ input: Buffer.from(svg),
1124
+ top: 0,
1125
+ left: 0
1126
+ }]).png().toBuffer();
1127
+ return {
1128
+ data: new Uint8Array(buf),
1129
+ width,
1130
+ height
1131
+ };
1132
+ }
1133
+ //#endregion
1134
+ //#region src/keys.ts
1135
+ /**
1136
+ * keys.ts — named Android keycode table for `android_press_key`.
1137
+ *
1138
+ * Maps human-readable key names to Android `KeyEvent` keycode integers.
1139
+ * The model may pass either a named key (`"home"`, `"back"`) or a bare
1140
+ * integer keycode; `resolveKey` normalizes both into a numeric keycode.
1141
+ *
1142
+ * @module @huanlin/dsh-plugin-android-use/src/keys
1143
+ */
1144
+ /** Named key → Android keycode. Source: android.view.KeyEvent constants. */
1145
+ const KEY_CODES = {
1146
+ home: 3,
1147
+ back: 4,
1148
+ call: 5,
1149
+ endcall: 6,
1150
+ "0": 7,
1151
+ "1": 8,
1152
+ "2": 9,
1153
+ "3": 10,
1154
+ "4": 11,
1155
+ "5": 12,
1156
+ "6": 13,
1157
+ "7": 14,
1158
+ "8": 15,
1159
+ "9": 16,
1160
+ star: 17,
1161
+ pound: 18,
1162
+ dpad_up: 19,
1163
+ dpad_down: 20,
1164
+ dpad_left: 21,
1165
+ dpad_right: 22,
1166
+ dpad_center: 23,
1167
+ volume_up: 24,
1168
+ volume_down: 25,
1169
+ power: 26,
1170
+ camera: 27,
1171
+ clear: 28,
1172
+ a: 29,
1173
+ b: 30,
1174
+ c: 31,
1175
+ d: 32,
1176
+ e: 33,
1177
+ f: 34,
1178
+ g: 35,
1179
+ h: 36,
1180
+ i: 37,
1181
+ j: 38,
1182
+ k: 39,
1183
+ l: 40,
1184
+ m: 41,
1185
+ n: 42,
1186
+ o: 43,
1187
+ p: 44,
1188
+ q: 45,
1189
+ r: 46,
1190
+ s: 47,
1191
+ t: 48,
1192
+ u: 49,
1193
+ v: 50,
1194
+ w: 51,
1195
+ x: 52,
1196
+ y: 53,
1197
+ z: 54,
1198
+ comma: 55,
1199
+ period: 56,
1200
+ alt_left: 57,
1201
+ alt_right: 58,
1202
+ shift_left: 59,
1203
+ shift_right: 60,
1204
+ tab: 61,
1205
+ space: 62,
1206
+ sym: 63,
1207
+ explorer: 64,
1208
+ envelope: 65,
1209
+ enter: 66,
1210
+ del: 67,
1211
+ backspace: 67,
1212
+ grave: 68,
1213
+ minus: 69,
1214
+ equals: 70,
1215
+ left_bracket: 71,
1216
+ right_bracket: 72,
1217
+ backslash: 73,
1218
+ semicolon: 74,
1219
+ apostrophe: 75,
1220
+ slash: 76,
1221
+ at: 77,
1222
+ num: 78,
1223
+ headsethook: 79,
1224
+ focus: 80,
1225
+ plus: 81,
1226
+ menu: 82,
1227
+ notification: 83,
1228
+ search: 84,
1229
+ media_play_pause: 85,
1230
+ media_stop: 86,
1231
+ media_next: 87,
1232
+ media_previous: 88,
1233
+ media_rewind: 89,
1234
+ media_fast_forward: 90,
1235
+ mute: 91,
1236
+ page_up: 92,
1237
+ page_down: 93,
1238
+ pict_symbols: 94,
1239
+ switch_charset: 95,
1240
+ button_a: 96,
1241
+ button_b: 97,
1242
+ button_c: 98,
1243
+ button_x: 99,
1244
+ button_y: 100,
1245
+ button_z: 101,
1246
+ button_l1: 102,
1247
+ button_r1: 103,
1248
+ button_l2: 104,
1249
+ button_r2: 105,
1250
+ button_thumbl: 106,
1251
+ button_thumbr: 107,
1252
+ button_start: 108,
1253
+ button_select: 109,
1254
+ button_mode: 110,
1255
+ escape: 111,
1256
+ forward_del: 112,
1257
+ delete: 112,
1258
+ ctrl_left: 113,
1259
+ ctrl_right: 114,
1260
+ caps_lock: 115,
1261
+ scroll_lock: 116,
1262
+ meta_left: 117,
1263
+ meta_right: 118,
1264
+ function: 119,
1265
+ sysrq: 120,
1266
+ break: 121,
1267
+ move_home: 122,
1268
+ move_end: 123,
1269
+ insert: 124,
1270
+ forward: 125,
1271
+ media_play: 126,
1272
+ media_pause: 127,
1273
+ media_close: 128,
1274
+ media_eject: 129,
1275
+ media_record: 130,
1276
+ f1: 131,
1277
+ f2: 132,
1278
+ f3: 133,
1279
+ f4: 134,
1280
+ f5: 135,
1281
+ f6: 136,
1282
+ f7: 137,
1283
+ f8: 138,
1284
+ f9: 139,
1285
+ f10: 140,
1286
+ f11: 141,
1287
+ f12: 142,
1288
+ num_lock: 143,
1289
+ numpad_0: 144,
1290
+ numpad_1: 145,
1291
+ numpad_2: 146,
1292
+ numpad_3: 147,
1293
+ numpad_4: 148,
1294
+ numpad_5: 149,
1295
+ numpad_6: 150,
1296
+ numpad_7: 151,
1297
+ numpad_8: 152,
1298
+ numpad_9: 153,
1299
+ numpad_add: 154,
1300
+ numpad_subtract: 155,
1301
+ numpad_multiply: 156,
1302
+ numpad_divide: 157,
1303
+ numpad_dot: 158,
1304
+ numpad_comma: 159,
1305
+ numpad_enter: 160,
1306
+ numpad_equals: 161,
1307
+ numpad_left_paren: 162,
1308
+ numpad_right_paren: 163,
1309
+ volume_mute: 164,
1310
+ info: 165,
1311
+ channel_up: 166,
1312
+ channel_down: 167,
1313
+ zoom_in: 168,
1314
+ zoom_out: 169,
1315
+ tv: 170,
1316
+ window: 171,
1317
+ guide: 172,
1318
+ dvr: 173,
1319
+ bookmark: 174,
1320
+ captions: 175,
1321
+ settings: 176,
1322
+ tv_power: 177,
1323
+ tv_input: 178,
1324
+ stb_power: 179,
1325
+ stb_input: 180,
1326
+ avr_power: 181,
1327
+ avr_input: 182,
1328
+ prog_red: 183,
1329
+ prog_green: 184,
1330
+ prog_yellow: 185,
1331
+ prog_blue: 186,
1332
+ app_switch: 187,
1333
+ button_1: 188,
1334
+ button_2: 189,
1335
+ button_3: 190,
1336
+ button_4: 191,
1337
+ button_5: 192,
1338
+ button_6: 193,
1339
+ button_7: 194,
1340
+ button_8: 195,
1341
+ button_9: 196,
1342
+ button_10: 197,
1343
+ button_11: 198,
1344
+ button_12: 199,
1345
+ button_13: 200,
1346
+ button_14: 201,
1347
+ button_15: 202,
1348
+ button_16: 203,
1349
+ language_switch: 204,
1350
+ manner_mode: 205,
1351
+ "3d_mode": 206,
1352
+ contacts: 207,
1353
+ calendar: 208,
1354
+ music: 209,
1355
+ calculator: 210,
1356
+ zenkaku_hankaku: 211,
1357
+ eisu: 212,
1358
+ muhenkan: 213,
1359
+ henkan: 214,
1360
+ katakana_hiragana: 215,
1361
+ yen: 216,
1362
+ ro: 217,
1363
+ kana: 218,
1364
+ assist: 219,
1365
+ brightness_down: 220,
1366
+ brightness_up: 221,
1367
+ media_audio_track: 222,
1368
+ sleep: 223,
1369
+ wakeup: 224,
1370
+ soft_sleep: 225,
1371
+ cut: 277,
1372
+ copy: 278,
1373
+ paste: 279,
1374
+ system_navigation_up: 280,
1375
+ system_navigation_down: 281,
1376
+ system_navigation_left: 282,
1377
+ system_navigation_right: 283,
1378
+ all_apps: 284,
1379
+ refresh: 285
1380
+ };
1381
+ /**
1382
+ * Resolve a key name or raw keycode into a numeric Android keycode.
1383
+ * @param input - named key (case-insensitive, e.g. `"home"`, `"BACK"`) or a
1384
+ * string/number representing a raw keycode integer.
1385
+ * @returns the numeric keycode.
1386
+ * @throws when the name is unknown and the input is not a valid integer.
1387
+ */
1388
+ function resolveKey(input) {
1389
+ if (typeof input === "number") {
1390
+ if (!Number.isInteger(input)) throw new Error(`keycode must be an integer, got ${input}`);
1391
+ return input;
1392
+ }
1393
+ const asNum = Number(input);
1394
+ if (input.trim() !== "" && Number.isInteger(asNum) && String(asNum) === input.trim()) return asNum;
1395
+ const lower = input.toLowerCase();
1396
+ const code = KEY_CODES[lower];
1397
+ if (code === void 0) throw new Error(`unknown key name "${input}"; pass a named key (e.g. "home", "back") or a raw integer keycode`);
1398
+ return code;
1399
+ }
1400
+ //#endregion
1401
+ //#region src/tools/input.ts
1402
+ /**
1403
+ * Escape text for `adb shell input text`. Spaces become `%s` and shell
1404
+ * metacharacters are backslash-escaped. Non-ASCII characters are rejected
1405
+ * (the `input` command cannot encode them).
1406
+ * @param text - the raw text to escape.
1407
+ * @returns the escaped text safe for `adb shell input text`.
1408
+ * @throws when the text contains non-ASCII characters.
1409
+ */
1410
+ function escapeInputText(text) {
1411
+ for (let i = 0; i < text.length; i++) if (text.charCodeAt(i) > 127) throw new Error(`the "input" text mode cannot encode non-ASCII character "${text[i]}" (U+${text.charCodeAt(i).toString(16).toUpperCase()}); set the plugin config inputTextMode to "adbkeyboard" to inject Unicode text via ADBKeyboard`);
1412
+ let out = "";
1413
+ for (let i = 0; i < text.length; i++) {
1414
+ const ch = text[i];
1415
+ switch (ch) {
1416
+ case " ":
1417
+ out += "%s";
1418
+ break;
1419
+ case "&":
1420
+ case "<":
1421
+ case ">":
1422
+ case ";":
1423
+ case "(":
1424
+ case ")":
1425
+ case "|":
1426
+ case "^":
1427
+ case "*":
1428
+ case "~":
1429
+ case "\"":
1430
+ case "'":
1431
+ case "`":
1432
+ case "$":
1433
+ case "!":
1434
+ case "#":
1435
+ out += "\\" + ch;
1436
+ break;
1437
+ case "\\":
1438
+ out += "\\\\";
1439
+ break;
1440
+ default: out += ch;
1441
+ }
1442
+ }
1443
+ return out;
1444
+ }
1445
+ /**
1446
+ * URL-encode text for ADBKeyboard broadcast injection (`am broadcast -a ADB_INPUT_TEXT --es msg <encoded>`).
1447
+ * @param text - the raw text to encode (any Unicode).
1448
+ * @returns the percent-encoded text.
1449
+ */
1450
+ function encodeAdbKeyboard(text) {
1451
+ return encodeURIComponent(text);
1452
+ }
1453
+ function renderTap(_args, value) {
1454
+ const v = value;
1455
+ const lines = [`Tap at image (${v.x}, ${v.y}) → device (${v.device_x}, ${v.device_y}) on ${v.serial}: ${v.action} ×${v.times}` + (v.duration_ms > 0 ? ` (${v.duration_ms}ms)` : "")];
1456
+ if (v.pre_tap_screenshot !== null) lines.push(`Pre-tap screenshot (annotated with tap position marker): ${v.pre_tap_screenshot.width}x${v.pre_tap_screenshot.height} px, ${v.pre_tap_screenshot.bytes} bytes.`);
1457
+ else lines.push("No pre-tap screenshot (attachment store unavailable or capture failed).");
1458
+ if (v.post_tap_screenshot !== null) lines.push(`Post-tap screenshot (showing the screen result after tap): ${v.post_tap_screenshot.width}x${v.post_tap_screenshot.height} px, ${v.post_tap_screenshot.bytes} bytes.`);
1459
+ else lines.push("No post-tap screenshot (attachment store unavailable or capture failed).");
1460
+ lines.push(`Images emitted to model: ${v.screenshot_emitted ? "yes" : "no"}.`);
1461
+ const blocks = [{
1462
+ type: "text",
1463
+ text: lines.join("\n")
1464
+ }];
1465
+ if (v.screenshot_emitted) {
1466
+ if (v.pre_tap_screenshot !== null) {
1467
+ blocks.push({
1468
+ type: "text",
1469
+ text: "Pre-tap screenshot (annotated with tap position marker):"
1470
+ });
1471
+ blocks.push({
1472
+ type: "image",
1473
+ attachment: {
1474
+ attachmentId: v.pre_tap_screenshot.attachmentId,
1475
+ mediaType: "image/png",
1476
+ bytes: v.pre_tap_screenshot.bytes,
1477
+ width: v.pre_tap_screenshot.width,
1478
+ height: v.pre_tap_screenshot.height
1479
+ }
1480
+ });
1481
+ }
1482
+ if (v.post_tap_screenshot !== null) {
1483
+ blocks.push({
1484
+ type: "text",
1485
+ text: "Post-tap screenshot (showing the screen result after tap):"
1486
+ });
1487
+ blocks.push({
1488
+ type: "image",
1489
+ attachment: {
1490
+ attachmentId: v.post_tap_screenshot.attachmentId,
1491
+ mediaType: "image/png",
1492
+ bytes: v.post_tap_screenshot.bytes,
1493
+ width: v.post_tap_screenshot.width,
1494
+ height: v.post_tap_screenshot.height
1495
+ }
1496
+ });
1497
+ }
1498
+ }
1499
+ return blocks;
1500
+ }
1501
+ function renderSwipe(value) {
1502
+ return `Swipe image (${value.x1}, ${value.y1}) → (${value.x2}, ${value.y2}) / device (${value.device_x1}, ${value.device_y1}) → (${value.device_x2}, ${value.device_y2}) on ${value.serial}` + (value.duration_ms > 0 ? ` over ${value.duration_ms}ms` : "");
1503
+ }
1504
+ function renderPressKey(value) {
1505
+ return `Pressed key "${value.key}" (keycode ${value.keycode}) on ${value.serial} ×${value.times}`;
1506
+ }
1507
+ function renderInputText(value) {
1508
+ const lines = [`Input text on ${value.serial} (mode: ${value.mode}):`, ` text: ${JSON.stringify(value.text)}`];
1509
+ if (value.submitted) lines.push(" submitted (Enter pressed)");
1510
+ return lines.join("\n");
1511
+ }
1512
+ function textRender$1(fn) {
1513
+ return (_args, value) => [{
1514
+ type: "text",
1515
+ text: fn(value)
1516
+ }];
1517
+ }
1518
+ /** Register `android_tap`, `android_swipe`, `android_press_key`, `android_input_text`. */
1519
+ function registerInputTools(ctx, deps) {
1520
+ ctx.tools.register(defineTool({
1521
+ name: "android_tap",
1522
+ description: "Tap a point on the Android screen. Pass (x, y) in screenshot image coordinates — the same coordinate space as the pixels in android_screenshot and the \"center\" values from android_ui_dump. The plugin automatically converts these to device-native coordinates for execution. Use `duration_ms` for a long-press (hold). Use `times` to repeat the tap. A pre-tap screenshot annotated with a marker at the tap position is captured and returned when an attachment store is available.",
1523
+ parameters: {
1524
+ x: {
1525
+ type: "integer",
1526
+ required: true,
1527
+ description: "X coordinate in screenshot image space."
1528
+ },
1529
+ y: {
1530
+ type: "integer",
1531
+ required: true,
1532
+ description: "Y coordinate in screenshot image space."
1533
+ },
1534
+ duration_ms: {
1535
+ type: "integer",
1536
+ description: "Hold duration in milliseconds. When > 0, performs a long-press (swipe-to-same-point) instead of a quick tap."
1537
+ },
1538
+ times: {
1539
+ type: "integer",
1540
+ description: "Number of times to repeat the tap (default 1)."
1541
+ },
1542
+ serial: {
1543
+ type: "string",
1544
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
1545
+ }
1546
+ },
1547
+ output: {
1548
+ schema: {
1549
+ type: "object",
1550
+ additionalProperties: false,
1551
+ properties: {
1552
+ serial: {
1553
+ type: "string",
1554
+ required: true
1555
+ },
1556
+ x: {
1557
+ type: "integer",
1558
+ required: true
1559
+ },
1560
+ y: {
1561
+ type: "integer",
1562
+ required: true
1563
+ },
1564
+ device_x: {
1565
+ type: "integer",
1566
+ required: true
1567
+ },
1568
+ device_y: {
1569
+ type: "integer",
1570
+ required: true
1571
+ },
1572
+ duration_ms: {
1573
+ type: "integer",
1574
+ required: true
1575
+ },
1576
+ times: {
1577
+ type: "integer",
1578
+ required: true
1579
+ },
1580
+ action: {
1581
+ type: "string",
1582
+ enum: ["tap", "long_press"],
1583
+ required: true
1584
+ },
1585
+ pre_tap_screenshot: {
1586
+ oneOf: [{ type: "null" }, {
1587
+ type: "object",
1588
+ additionalProperties: false,
1589
+ properties: {
1590
+ attachmentId: {
1591
+ type: "string",
1592
+ required: true
1593
+ },
1594
+ bytes: {
1595
+ type: "integer",
1596
+ required: true
1597
+ },
1598
+ width: {
1599
+ type: "integer",
1600
+ required: true
1601
+ },
1602
+ height: {
1603
+ type: "integer",
1604
+ required: true
1605
+ }
1606
+ }
1607
+ }],
1608
+ required: true
1609
+ },
1610
+ post_tap_screenshot: {
1611
+ oneOf: [{ type: "null" }, {
1612
+ type: "object",
1613
+ additionalProperties: false,
1614
+ properties: {
1615
+ attachmentId: {
1616
+ type: "string",
1617
+ required: true
1618
+ },
1619
+ bytes: {
1620
+ type: "integer",
1621
+ required: true
1622
+ },
1623
+ width: {
1624
+ type: "integer",
1625
+ required: true
1626
+ },
1627
+ height: {
1628
+ type: "integer",
1629
+ required: true
1630
+ }
1631
+ }
1632
+ }],
1633
+ required: true
1634
+ },
1635
+ screenshot_emitted: {
1636
+ type: "boolean",
1637
+ required: true
1638
+ }
1639
+ }
1640
+ },
1641
+ render: renderTap
1642
+ },
1643
+ async execute(args, exec) {
1644
+ const a = args;
1645
+ const config = deps.getConfig();
1646
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
1647
+ const dur = typeof a.duration_ms === "number" && a.duration_ms > 0 ? a.duration_ms : 0;
1648
+ const times = typeof a.times === "number" && a.times > 0 ? a.times : 1;
1649
+ const action = dur > 0 ? "long_press" : "tap";
1650
+ let preScreenshot = null;
1651
+ let postScreenshot = null;
1652
+ let screenshotEmitted = false;
1653
+ const attachments = ctx.get("attachments");
1654
+ let scaleX = 1;
1655
+ let scaleY = 1;
1656
+ if (attachments !== void 0) try {
1657
+ const raw = await deps.adb.execOutBinary(serial, "screencap -p", exec.signal);
1658
+ const fitted = await fitToMaxDimension(new Uint8Array(raw), attachments.imageLimits.maxImageDimension);
1659
+ scaleX = fitted.scaleX;
1660
+ scaleY = fitted.scaleY;
1661
+ const annotated = await annotateTap(fitted.data, a.x, a.y);
1662
+ const ref = await attachments.saveImage({
1663
+ data: annotated.data,
1664
+ mediaType: "image/png",
1665
+ name: "pre_tap_screenshot.png"
1666
+ });
1667
+ preScreenshot = {
1668
+ attachmentId: ref.attachmentId,
1669
+ bytes: ref.bytes,
1670
+ width: ref.width,
1671
+ height: ref.height
1672
+ };
1673
+ } catch {}
1674
+ const deviceX = Math.round(a.x / scaleX);
1675
+ const deviceY = Math.round(a.y / scaleY);
1676
+ for (let i = 0; i < times; i++) if (dur > 0) await deps.adb.shell(serial, `input swipe ${deviceX} ${deviceY} ${deviceX} ${deviceY} ${dur}`, exec.signal);
1677
+ else await deps.adb.shell(serial, `input tap ${deviceX} ${deviceY}`, exec.signal);
1678
+ if (attachments !== void 0) try {
1679
+ await new Promise((resolve) => setTimeout(resolve, 500));
1680
+ const raw = await deps.adb.execOutBinary(serial, "screencap -p", exec.signal);
1681
+ const fitted = await fitToMaxDimension(new Uint8Array(raw), attachments.imageLimits.maxImageDimension);
1682
+ const ref = await attachments.saveImage({
1683
+ data: fitted.data,
1684
+ mediaType: "image/png",
1685
+ name: "post_tap_screenshot.png"
1686
+ });
1687
+ postScreenshot = {
1688
+ attachmentId: ref.attachmentId,
1689
+ bytes: ref.bytes,
1690
+ width: ref.width,
1691
+ height: ref.height
1692
+ };
1693
+ screenshotEmitted = await routeIsImageCapable(ctx, exec, exec.signal);
1694
+ } catch {}
1695
+ return {
1696
+ serial,
1697
+ x: a.x,
1698
+ y: a.y,
1699
+ device_x: deviceX,
1700
+ device_y: deviceY,
1701
+ duration_ms: dur,
1702
+ times,
1703
+ action,
1704
+ pre_tap_screenshot: preScreenshot,
1705
+ post_tap_screenshot: postScreenshot,
1706
+ screenshot_emitted: screenshotEmitted
1707
+ };
1708
+ }
1709
+ }));
1710
+ ctx.tools.register(defineTool({
1711
+ name: "android_swipe",
1712
+ description: "Swipe from one point to another on the Android screen. Pass start and end coordinates in screenshot image space (the same coordinate space as android_screenshot and android_ui_dump). The plugin automatically converts these to device-native coordinates for execution. Use `duration_ms` to control swipe speed (longer = slower).",
1713
+ parameters: {
1714
+ x1: {
1715
+ type: "integer",
1716
+ required: true,
1717
+ description: "Start X coordinate in screenshot image space."
1718
+ },
1719
+ y1: {
1720
+ type: "integer",
1721
+ required: true,
1722
+ description: "Start Y coordinate in screenshot image space."
1723
+ },
1724
+ x2: {
1725
+ type: "integer",
1726
+ required: true,
1727
+ description: "End X coordinate in screenshot image space."
1728
+ },
1729
+ y2: {
1730
+ type: "integer",
1731
+ required: true,
1732
+ description: "End Y coordinate in screenshot image space."
1733
+ },
1734
+ duration_ms: {
1735
+ type: "integer",
1736
+ description: "Swipe duration in milliseconds (default 300). Longer values produce slower swipes."
1737
+ },
1738
+ serial: {
1739
+ type: "string",
1740
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
1741
+ }
1742
+ },
1743
+ output: {
1744
+ schema: {
1745
+ type: "object",
1746
+ additionalProperties: false,
1747
+ properties: {
1748
+ serial: {
1749
+ type: "string",
1750
+ required: true
1751
+ },
1752
+ x1: {
1753
+ type: "integer",
1754
+ required: true
1755
+ },
1756
+ y1: {
1757
+ type: "integer",
1758
+ required: true
1759
+ },
1760
+ x2: {
1761
+ type: "integer",
1762
+ required: true
1763
+ },
1764
+ y2: {
1765
+ type: "integer",
1766
+ required: true
1767
+ },
1768
+ device_x1: {
1769
+ type: "integer",
1770
+ required: true
1771
+ },
1772
+ device_y1: {
1773
+ type: "integer",
1774
+ required: true
1775
+ },
1776
+ device_x2: {
1777
+ type: "integer",
1778
+ required: true
1779
+ },
1780
+ device_y2: {
1781
+ type: "integer",
1782
+ required: true
1783
+ },
1784
+ duration_ms: {
1785
+ type: "integer",
1786
+ required: true
1787
+ }
1788
+ }
1789
+ },
1790
+ render: textRender$1(renderSwipe)
1791
+ },
1792
+ async execute(args, exec) {
1793
+ const a = args;
1794
+ const config = deps.getConfig();
1795
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
1796
+ const dur = typeof a.duration_ms === "number" && a.duration_ms > 0 ? a.duration_ms : 300;
1797
+ const attachments = ctx.get("attachments");
1798
+ let scale = 1;
1799
+ if (attachments !== void 0) {
1800
+ const { width: devW, height: devH } = parseWmSize$1(await deps.adb.shell(serial, "wm size", exec.signal));
1801
+ scale = computeScale(devW, devH, attachments.imageLimits.maxImageDimension);
1802
+ }
1803
+ const dx1 = Math.round(a.x1 / scale);
1804
+ const dy1 = Math.round(a.y1 / scale);
1805
+ const dx2 = Math.round(a.x2 / scale);
1806
+ const dy2 = Math.round(a.y2 / scale);
1807
+ await deps.adb.shell(serial, `input swipe ${dx1} ${dy1} ${dx2} ${dy2} ${dur}`, exec.signal);
1808
+ return {
1809
+ serial,
1810
+ x1: a.x1,
1811
+ y1: a.y1,
1812
+ x2: a.x2,
1813
+ y2: a.y2,
1814
+ device_x1: dx1,
1815
+ device_y1: dy1,
1816
+ device_x2: dx2,
1817
+ device_y2: dy2,
1818
+ duration_ms: dur
1819
+ };
1820
+ }
1821
+ }));
1822
+ ctx.tools.register(defineTool({
1823
+ name: "android_press_key",
1824
+ description: "Press a hardware/key event key on the Android device. Pass a named key (case-insensitive: \"home\", \"back\", \"app_switch\", \"power\", \"enter\", \"volume_up\", \"volume_down\", \"mute\", \"camera\", \"search\", \"menu\", \"escape\", \"delete\", \"tab\", \"space\", \"dpad_up\", \"dpad_down\", \"dpad_left\", \"dpad_right\", \"dpad_center\", etc.) or a raw integer keycode. Use `times` to repeat.",
1825
+ parameters: {
1826
+ key: {
1827
+ oneOf: [{
1828
+ type: "string",
1829
+ description: "Named key (case-insensitive), e.g. \"home\", \"back\", \"app_switch\"."
1830
+ }, {
1831
+ type: "integer",
1832
+ description: "Raw Android keycode integer."
1833
+ }],
1834
+ required: true,
1835
+ description: "Key to press: a named key or a raw integer keycode."
1836
+ },
1837
+ times: {
1838
+ type: "integer",
1839
+ description: "Number of times to repeat the key press (default 1)."
1840
+ },
1841
+ serial: {
1842
+ type: "string",
1843
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
1844
+ }
1845
+ },
1846
+ output: {
1847
+ schema: {
1848
+ type: "object",
1849
+ additionalProperties: false,
1850
+ properties: {
1851
+ serial: {
1852
+ type: "string",
1853
+ required: true
1854
+ },
1855
+ key: {
1856
+ type: "string",
1857
+ required: true
1858
+ },
1859
+ keycode: {
1860
+ type: "integer",
1861
+ required: true
1862
+ },
1863
+ times: {
1864
+ type: "integer",
1865
+ required: true
1866
+ }
1867
+ }
1868
+ },
1869
+ render: textRender$1(renderPressKey)
1870
+ },
1871
+ async execute(args, exec) {
1872
+ const a = args;
1873
+ const config = deps.getConfig();
1874
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
1875
+ const keycode = resolveKey(a.key);
1876
+ const times = typeof a.times === "number" && a.times > 0 ? a.times : 1;
1877
+ const keyStr = typeof a.key === "number" ? String(a.key) : a.key;
1878
+ for (let i = 0; i < times; i++) await deps.adb.shell(serial, `input keyevent ${keycode}`, exec.signal);
1879
+ return {
1880
+ serial,
1881
+ key: keyStr,
1882
+ keycode,
1883
+ times
1884
+ };
1885
+ }
1886
+ }));
1887
+ ctx.tools.register(defineTool({
1888
+ name: "android_input_text",
1889
+ description: "Type text into the focused input field on the Android device. In \"input\" mode (default), only ASCII text is supported — non-ASCII characters require switching the plugin config `inputTextMode` to \"adbkeyboard\" (requires the ADBKeyboard IME installed on the device). Pass `submit: true` to press Enter after typing.",
1890
+ parameters: {
1891
+ text: {
1892
+ type: "string",
1893
+ required: true,
1894
+ description: "Text to type into the focused field."
1895
+ },
1896
+ submit: {
1897
+ type: "boolean",
1898
+ description: "When true, presses Enter (keycode 66) after typing the text."
1899
+ },
1900
+ serial: {
1901
+ type: "string",
1902
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
1903
+ }
1904
+ },
1905
+ output: {
1906
+ schema: {
1907
+ type: "object",
1908
+ additionalProperties: false,
1909
+ properties: {
1910
+ serial: {
1911
+ type: "string",
1912
+ required: true
1913
+ },
1914
+ text: {
1915
+ type: "string",
1916
+ required: true
1917
+ },
1918
+ mode: {
1919
+ type: "string",
1920
+ enum: ["input", "adbkeyboard"],
1921
+ required: true
1922
+ },
1923
+ submitted: {
1924
+ type: "boolean",
1925
+ required: true
1926
+ }
1927
+ }
1928
+ },
1929
+ render: textRender$1(renderInputText)
1930
+ },
1931
+ async execute(args, exec) {
1932
+ const a = args;
1933
+ const config = deps.getConfig();
1934
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
1935
+ const mode = config.inputTextMode;
1936
+ const submitted = a.submit === true;
1937
+ if (mode === "adbkeyboard") {
1938
+ const encoded = encodeAdbKeyboard(a.text);
1939
+ await deps.adb.shell(serial, `am broadcast -a ADB_INPUT_TEXT --es msg "${encoded}"`, exec.signal);
1940
+ } else {
1941
+ const escaped = escapeInputText(a.text);
1942
+ await deps.adb.shell(serial, `input text "${escaped}"`, exec.signal);
1943
+ }
1944
+ if (submitted) await deps.adb.shell(serial, "input keyevent 66", exec.signal);
1945
+ return {
1946
+ serial,
1947
+ text: a.text,
1948
+ mode,
1949
+ submitted
1950
+ };
1951
+ }
1952
+ }));
1953
+ }
1954
+ //#endregion
1955
+ //#region src/tools/apps.ts
1956
+ /**
1957
+ * Parse `dumpsys window` output for the current focus window.
1958
+ *
1959
+ * Example line: ` mCurrentFocus=Window{f6144b2 u0 com.android.launcher/com.android.launcher.Launcher}`
1960
+ * Also handles `mCurrentFocus=null` (display off or no focused window).
1961
+ * @param output - the grep-filtered dumpsys window output.
1962
+ * @returns the package and activity, or null when no window is focused.
1963
+ */
1964
+ function parseForegroundApp(output) {
1965
+ const lines = output.split(/\r?\n/);
1966
+ let lastFocus = null;
1967
+ for (const line of lines) {
1968
+ const match = /mCurrentFocus=(.+)/.exec(line.trim());
1969
+ if (match !== null) lastFocus = match[1];
1970
+ }
1971
+ if (lastFocus === null) return null;
1972
+ if (lastFocus === "null") return null;
1973
+ const windowMatch = /Window\{[^}]*?\s+([^/\s}]+)\/([^\s}]+)\s*\}/.exec(lastFocus);
1974
+ if (windowMatch !== null) return {
1975
+ package: windowMatch[1],
1976
+ activity: windowMatch[2],
1977
+ windowTitle: null
1978
+ };
1979
+ const pairMatch = /([a-zA-Z0-9_.]+)\/([a-zA-Z0-9_.]+)/.exec(lastFocus);
1980
+ if (pairMatch !== null) return {
1981
+ package: pairMatch[1],
1982
+ activity: pairMatch[2],
1983
+ windowTitle: null
1984
+ };
1985
+ return {
1986
+ package: lastFocus,
1987
+ activity: "",
1988
+ windowTitle: lastFocus
1989
+ };
1990
+ }
1991
+ /** Parse `dumpsys power` output for the wakefulness state. */
1992
+ function parseWakefulness(output) {
1993
+ return /mWakefulness=Awake/.test(output);
1994
+ }
1995
+ function renderOpenApp(value) {
1996
+ return `Opened app ${value.activity !== void 0 ? `${value.package}/${value.activity}` : value.package} on ${value.serial}: ${value.started ? "started" : "failed"}`;
1997
+ }
1998
+ function renderForegroundApp(value) {
1999
+ if (value.package === null) return `No focused window on ${value.serial} (screen on: ${value.screen_on}).`;
2000
+ const lines = [
2001
+ `Foreground app on ${value.serial}:`,
2002
+ ` package: ${value.package}`,
2003
+ ` activity: ${value.activity ?? "(unknown)"}`
2004
+ ];
2005
+ if (value.window_title !== null) lines.push(` window: ${value.window_title}`);
2006
+ lines.push(` screen on: ${value.screen_on}`);
2007
+ return lines.join("\n");
2008
+ }
2009
+ function textRender(fn) {
2010
+ return (_args, value) => [{
2011
+ type: "text",
2012
+ text: fn(value)
2013
+ }];
2014
+ }
2015
+ /** Register `android_open_app` and `android_foreground_app`. */
2016
+ function registerAppTools(ctx, deps) {
2017
+ ctx.tools.register(defineTool({
2018
+ name: "android_open_app",
2019
+ description: "Open an app on the Android device by package name. If you know the specific activity, pass it; otherwise the app's default launcher activity is started via monkey. Example packages: \"com.android.settings\", \"com.android.chrome\", \"com.tencent.mm\" (WeChat).",
2020
+ parameters: {
2021
+ package: {
2022
+ type: "string",
2023
+ required: true,
2024
+ description: "Android package name, e.g. \"com.android.settings\"."
2025
+ },
2026
+ activity: {
2027
+ type: "string",
2028
+ description: "Specific activity to start (e.g. \".Settings\"). Omit to launch the app's default activity."
2029
+ }
2030
+ },
2031
+ output: {
2032
+ schema: {
2033
+ type: "object",
2034
+ additionalProperties: false,
2035
+ properties: {
2036
+ serial: {
2037
+ type: "string",
2038
+ required: true
2039
+ },
2040
+ package: {
2041
+ type: "string",
2042
+ required: true
2043
+ },
2044
+ activity: { type: "string" },
2045
+ started: {
2046
+ type: "boolean",
2047
+ required: true
2048
+ }
2049
+ }
2050
+ },
2051
+ render: textRender(renderOpenApp)
2052
+ },
2053
+ async execute(args, exec) {
2054
+ const a = args;
2055
+ const config = deps.getConfig();
2056
+ const { serial } = await resolveSerial(deps.adb, void 0, config.defaultSerial, exec.signal);
2057
+ if (a.activity !== void 0 && a.activity !== "") {
2058
+ const fullActivity = a.activity.startsWith(".") ? `${a.package}${a.activity}` : a.activity;
2059
+ await deps.adb.shell(serial, `am start -n ${a.package}/${fullActivity}`, exec.signal);
2060
+ } else await deps.adb.shell(serial, `monkey -p ${a.package} -c android.intent.category.LAUNCHER 1`, exec.signal);
2061
+ const result = {
2062
+ serial,
2063
+ package: a.package,
2064
+ started: true
2065
+ };
2066
+ if (a.activity !== void 0 && a.activity !== "") result.activity = a.activity;
2067
+ return result;
2068
+ }
2069
+ }));
2070
+ ctx.tools.register(defineTool({
2071
+ name: "android_foreground_app",
2072
+ description: "Get the currently focused/foreground app and activity on the Android device. Also reports whether the screen is on. Useful for verifying which app is visible after opening an app or pressing home.",
2073
+ parameters: { serial: {
2074
+ type: "string",
2075
+ description: "Device serial. Omit when only one device is attached; required when multiple are connected."
2076
+ } },
2077
+ output: {
2078
+ schema: {
2079
+ type: "object",
2080
+ additionalProperties: false,
2081
+ properties: {
2082
+ serial: {
2083
+ type: "string",
2084
+ required: true
2085
+ },
2086
+ package: {
2087
+ oneOf: [{ type: "null" }, { type: "string" }],
2088
+ required: true
2089
+ },
2090
+ activity: {
2091
+ oneOf: [{ type: "null" }, { type: "string" }],
2092
+ required: true
2093
+ },
2094
+ window_title: {
2095
+ oneOf: [{ type: "null" }, { type: "string" }],
2096
+ required: true
2097
+ },
2098
+ screen_on: {
2099
+ type: "boolean",
2100
+ required: true
2101
+ }
2102
+ }
2103
+ },
2104
+ render: textRender(renderForegroundApp)
2105
+ },
2106
+ async execute(args, exec) {
2107
+ const a = args;
2108
+ const config = deps.getConfig();
2109
+ const { serial } = await resolveSerial(deps.adb, a.serial, config.defaultSerial, exec.signal);
2110
+ const parts = (await deps.adb.shell(serial, "echo \"=FOCUS=\"; dumpsys window|grep mCurrentFocus; echo \"=WAKE=\"; dumpsys power|grep mWakefulness", exec.signal)).split(/=[A-Z]+=/);
2111
+ const focusOutput = parts[1] ?? "";
2112
+ const wakeOutput = parts[2] ?? "";
2113
+ const parsed = parseForegroundApp(focusOutput);
2114
+ const screenOn = parseWakefulness(wakeOutput);
2115
+ return {
2116
+ serial,
2117
+ package: parsed?.package ?? null,
2118
+ activity: parsed?.activity ?? null,
2119
+ window_title: parsed?.windowTitle ?? null,
2120
+ screen_on: screenOn
2121
+ };
2122
+ }
2123
+ }));
2124
+ }
2125
+ //#endregion
2126
+ //#region src/registry.ts
2127
+ /**
2128
+ * Register all 10 android_* tools into the given context.
2129
+ * @param ctx - the plugin context (provides `ctx.tools.register`).
2130
+ * @param deps - the AdbClient and live config getter.
2131
+ */
2132
+ function registerTools(ctx, deps) {
2133
+ registerDeviceTools(ctx, deps);
2134
+ registerScreenTools(ctx, deps);
2135
+ registerInputTools(ctx, deps);
2136
+ registerAppTools(ctx, deps);
2137
+ }
2138
+ //#endregion
2139
+ //#region src/index.ts
2140
+ /**
2141
+ * index.ts — dsh-android-use cordis plugin entry (host half).
2142
+ *
2143
+ * 10 model-facing tools that let the AI operate an Android phone via adb:
2144
+ * - android_list_devices / android_device_info
2145
+ * - android_screenshot / android_ui_dump
2146
+ * - android_tap / android_swipe / android_press_key / android_input_text
2147
+ * - android_open_app / android_foreground_app
2148
+ *
2149
+ * Host-only (no client UI); the generic tool card is used for rendering.
2150
+ * Tool registration is effect-based: disposing the plugin fiber
2151
+ * (e.g., on config change) automatically unregisters all tools, and the
2152
+ * next apply() re-registers with the fresh config.
2153
+ *
2154
+ * @module @huanlin/dsh-plugin-android-use
2155
+ */
2156
+ const name = "dsh-android-use";
2157
+ const inject = ["tools"];
2158
+ const Config = z.object({
2159
+ adbPath: z.string().default("adb").description("Path to the adb binary. Defaults to \"adb\" (must be on PATH)."),
2160
+ defaultSerial: z.string().description("Default device serial. Omit to auto-select when one device is attached; required when multiple are connected."),
2161
+ inputTextMode: z.union(["input", "adbkeyboard"]).default("input").description("Text input mode: \"input\" (ASCII only, uses `adb shell input text`) or \"adbkeyboard\" (Unicode, requires ADBKeyboard IME on device).")
2162
+ });
2163
+ function resolveConfig(config) {
2164
+ return {
2165
+ adbPath: typeof config.adbPath === "string" && config.adbPath !== "" ? config.adbPath : "adb",
2166
+ defaultSerial: typeof config.defaultSerial === "string" && config.defaultSerial !== "" ? config.defaultSerial : void 0,
2167
+ inputTextMode: config.inputTextMode === "adbkeyboard" ? "adbkeyboard" : "input"
2168
+ };
2169
+ }
2170
+ function apply(ctx, config = {}) {
2171
+ const resolved = resolveConfig(config);
2172
+ registerTools(ctx, {
2173
+ adb: new AdbClient(resolved.adbPath),
2174
+ getConfig: () => resolved
2175
+ });
2176
+ }
2177
+ //#endregion
2178
+ export { Config, apply, inject, name, resolveConfig };