@flxapps/adbax 0.0.0-stage → 0.1.0

package/docs/commands.md

@@ -0,0 +1,163 @@
+# Commands
+
+`adbax <command> [arguments] [flags]`. Flags go anywhere on the line,
+`--flag value` and `--flag=value` both work, and `--` ends flag parsing.
+The exception is `shell`: everything after it goes to the device as it is, so
+`-s` and the other global flags go before it.
+
+Exit status is 0 on success, 2 when something was not there (no match, an
+ambiguous selector, a `wait` or `scroll-to` that timed out) and 1 for any other
+failure. Errors go to stderr as `adbax: <message>`.
+
+## Global flags
+
+| Flag | |
+| --- | --- |
+| `-s <serial>` | the device to talk to; otherwise `$ANDROID_SERIAL`, then the pinned device, then the only one online |
+| `--ui` | after an action, print the screen it led to (saves a separate `adbax ui`) |
+| `--settle <ms>` | how long to give the screen after an action before reading it. Default: 150 with the server, which then also waits for events to stop; 700 without |
+| `--nth N` / `--first` | pick one of several matches instead of exiting 2 |
+| `--re` | the selector is a regular expression |
+| `--shot` | `x,y` are pixels of the last screenshot, not of the screen |
+| `--no-server` | plain `adb shell` and `uiautomator dump`, as on devices the server cannot run on |
+
+## Selectors
+
+Anything that takes a selector accepts:
+
+- **text**: matched against text, content-desc, hint and resource-id,
+  case-insensitively. Exact beats prefix beats substring; text beats
+  content-desc beats ids. A tap prefers clickable nodes, and a match that is not
+  clickable itself resolves to its nearest clickable ancestor.
+- **a row number**, `5` or `#5`: row 5 of the last `adbax ui`, `find`, `wait` or
+  `scroll-to`, without reading the screen again. Valid until the screen changes.
+- **coordinates**, `540,1200`: taken as they are, or scaled from the last
+  screenshot with `--shot`.
+
+## Devices
+
+| Command | |
+| --- | --- |
+| `devices` | attached devices; `*` marks the pinned one |
+| `use <serial>` | pin a device for later calls; `use -` clears the pin |
+| `avds` | the emulator's AVDs |
+| `boot <avd> [--cold] [--wipe] [--no-window]` | start an AVD, wait for `sys.boot_completed`, unlock and pin it. Emulator output goes to `~/.cache/adbax/emulator-<avd>.log`; gives up after six minutes |
+| `kill [--all]` | shut down the current emulator, or every running one |
+
+## Seeing the screen
+
+| Command | |
+| --- | --- |
+| `ui` | the screen as rows: index, kind, label, short resource-id, centre, size, state. The first line names the top activity, the screen size and the row count |
+| `ui --all` | every node, including labels already folded into a clickable parent |
+| `ui --full` | adds each node's class |
+| `ui --json` | the rows as JSON, for scripts |
+| `ui --raw` | the XML as it came off the device |
+| `ui --windows` | every window, not just the active one: status bar, navigation bar, the keyboard, overlays. Needs the server |
+| `find <text>` | only the nodes matching a selector, with their row numbers |
+| `inspect <text>` | every attribute of the best match, and its ancestor chain |
+| `current` | the focused window and the resumed activity |
+| `screenshot [path]` | writes a PNG (a `.jpg` path writes JPEG) and prints the path. Defaults to `~/.cache/adbax/shot.png` |
+| `screenshot --small` | 540 px wide JPEG. For Claude that is about 690 image tokens, where a full-size 1080x1920 screenshot is about 1,840 |
+| `screenshot --width N` | any width, scaled on the device |
+| `screenshot --quality N` | JPEG quality, 80 by default |
+| `screenshot --grid [step]` | a labelled coordinate grid over the image, in the image's own pixels (default: a sixth of its width). Needs the server |
+| `record [secs] [path]` | a screen recording, 15 s by default, 180 at most |
+
+Row kinds: `TAP` clickable, `EDIT` a text field, `CHK[x]`/`CHK[ ]` checkable,
+`SCRL` a scroll container, `TXT` text you can read but not act on. State marks:
+`*focused`, `*selected`, `*password`, `*scrollable`, `!disabled`.
+
+## Acting
+
+| Command | |
+| --- | --- |
+| `tap <selector>` | a tap at the centre of the match |
+| `longpress <selector> [--dur ms]` | press and hold, 900 ms by default |
+| `type <text>` | types into the focused field |
+| `type <text> --into <selector>` | taps the field first and waits until it has input focus |
+| `type --clear` | empties the field before typing (or on its own) |
+| `type <text> --set` | replaces the whole field in one step (ACTION_SET_TEXT), no keystrokes. For long values, where typing over the old text can leave part of it behind |
+| `type ... --enter` | presses Enter afterwards |
+| `type ... --no-verify` | skips reading the field back |
+| `key <key>...` | key events by name (`back home enter recents del tab esc up down left right menu search power volup voldown space paste`...), by number, or as `KEYCODE_*` |
+| `key <key> --long` | a long press of the key |
+| `back`, `home`, `recents` | shorthands |
+| `hide-keyboard` | presses Back only if the soft keyboard is showing; a plain `back` would leave the screen when it isn't |
+| `swipe <up\|down\|left\|right>` | a swipe across the screen. "up" is the finger moving up, which scrolls the content down |
+| `swipe <dir> --in <selector>` | the same inside one node, a carousel or a list |
+| `swipe x1,y1,x2,y2 [--dur ms]` | between two points, 300 ms by default |
+| `scroll-to <text> [--max 12] [--dir up\|down] [--tap]` | swipes the largest scroll container until the text shows up, then optionally taps it |
+| `wait <text> [--timeout 15] [--gone]` | blocks until the text appears (or disappears) |
+
+`type` sends key events where the virtual keyboard map has the characters and
+sets the field's text through `ACTION_SET_TEXT` where it does not, so `Grüße`
+and `日本` arrive intact with the server. Without it, `input text` drops
+non-ASCII on most builds and adbax warns about it. After typing, `type` reads the
+field back and says so when it holds something else, a sign that an IME
+autocorrected it or that characters went missing.
+
+## Apps
+
+| Command | |
+| --- | --- |
+| `apps [filter] [--system]` | installed packages, third-party only unless `--system` |
+| `launch <package>` | starts the launcher activity and returns once it is drawn |
+| `start <package/.Activity>` | starts one activity |
+| `stop <package>` | force-stop |
+| `clear <package>` | wipe the app's data |
+| `install <apk> [split.apk ...]` | `-r -g -t`: replace, grant runtime permissions, allow test builds. Several files are installed as one split APK |
+| `uninstall <package>` | |
+| `deeplink <url> [package]` | an `ACTION_VIEW` intent |
+| `perm <grant\|revoke> <package> <PERMISSION>...` | `CAMERA` is short for `android.permission.CAMERA` |
+| `perms <package>` | runtime permissions and their state |
+
+## Debugging
+
+| Command | |
+| --- | --- |
+| `log [package] [--lines 200] [--grep re] [--level W]` | logcat; with a package, only its current process |
+| `crash` | the crash buffer, which survives the process dying |
+| `clearlog` | empties the main and crash buffers |
+
+## Device state
+
+| Command | |
+| --- | --- |
+| `dark <on\|off>` | night mode |
+| `font <scale>` | system font scale, `1.0` normal, `1.3` large, `2.0` the maximum |
+| `anim <on\|off>` | all three animation scales; off also makes reads after actions settle sooner |
+| `rotate <0\|90\|180\|270\|auto>` | |
+| `demo <on\|off>` | SystemUI demo mode: 10:00, full battery and signal, no notification icons |
+
+## The server
+
+| Command | |
+| --- | --- |
+| `server status` | pid, Android version, uptime and the forwarded port |
+| `server start [--idle seconds]` | start it now rather than on first use |
+| `server stop` | stop it and free the accessibility connection |
+| `server restart` | |
+| `server log` | the server's stderr, from `/data/local/tmp/adbax-server.log` |
+
+## Raw adb
+
+| Command | |
+| --- | --- |
+| `shell <command>` | runs on the device shell |
+| `pull <remote> [local]`, `push <local> <remote>` | |
+| `version` | |
+
+`text`, `press`, `screencap`, `logcat` and `ls` are aliases of `type`,
+`longpress`, `screenshot`, `log` and `apps`.
+
+## Environment
+
+| Variable | |
+| --- | --- |
+| `ANDROID_SERIAL` | the device to use; `-s` and `adbax use` do the same |
+| `ANDROID_HOME` | the SDK, if adb or the emulator is not on the PATH |
+| `ADBAX_SERVER=0` | never start the on-device server |
+| `ADBAX_SERVER_IDLE` | seconds before an unused server exits (600) |
+| `ADBAX_CACHE` | where the device pin and the last dump go (`~/.cache/adbax`) |
+| `ADBAX_TRACE=1` | print every adb call and server request with its duration |

package/package.json

@@ -1,6 +1,20 @@
 {
   "name": "@flxapps/adbax",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Drive an Android device or emulator through its accessibility tree instead of screenshots and pixel coordinates.",
+  "keywords": ["android", "adb", "emulator", "uiautomator", "accessibility", "automation", "testing", "cli", "agent"],
+  "license": "Apache-2.0",
+  "author": "Felix Heller",
+  "homepage": "https://github.com/flxapps/adbax#readme",
+  "repository": { "type": "git", "url": "git+https://github.com/flxapps/adbax.git" },
+  "bugs": { "url": "https://github.com/flxapps/adbax/issues" },
+  "type": "module",
+  "bin": { "adbax": "bin/adbax.js" },
+  "files": ["bin/", "src/", "server/adbax-server.jar", "skill/", "docs/", "NOTICE"],
+  "engines": { "node": ">=18" },
+  "publishConfig": { "access": "public" },
+  "scripts": {
+    "test": "node --test",
+    "build:server": "sh server/build.sh"
+  }
+}

package/skill/SKILL.md

@@ -0,0 +1,143 @@
+---
+name: android-device
+description: Drive an Android emulator or physical device through its accessibility tree instead of screenshots and pixel guessing. Boot AVDs, read the screen as a labelled node list, tap and type by label, scroll to off-screen elements, wait for state, install APKs, read logcat and crashes, play canvas games through grid screenshots, set dark mode / font scale / demo-mode status bar for store screenshots. Use whenever a task involves an emulator or a connected phone, e.g. "open the app on the emulator", "tap through onboarding", "boot Pixel_7_Pro and check the settings screen", "why does it crash on launch", "take store screenshots".
+---
+
+# Driving an Android device
+
+`adbax` must be on the PATH (`npm install -g @flxapps/adbax`, or a link to
+`bin/adbax.js` in a checkout). `adbax help` prints the whole surface.
+
+## Read the tree, do not read pixels
+
+A full-size phone screenshot costs about 1,800 image tokens, has to be read
+like an image and gives coordinates you then guess at. The accessibility tree
+costs a few hundred text tokens and gives exact labels, ids and centres:
+
+```
+$ adbax ui
+# com.android.settings/.Settings   1080x1920   11 nodes
+  [0]  TAP     "Search settings"  id=search_action_bar  @540,147  1038x126
+  [5]  TAP     "Network & Internet / Wi‑Fi, mobile, data usage"  id=dashboard_tile  @540,903  1080x189
+```
+
+Then act on the label, never on the pixel:
+
+```bash
+adbax tap "Network & Internet"
+```
+
+The first call starts a small server on the device (about a second). After
+that a read takes 0.2-0.3 s end to end, so reading after every step is cheap.
+If adbax says the server is unavailable it has fallen back to `uiautomator dump`
+at about 2 s per read; budget for that and do not poll `ui` in a loop.
+
+## The loop
+
+Chain steps in **one** Bash call; each `adbax` is its own process and `&&` costs
+nothing:
+
+```bash
+adbax launch com.example.app && adbax wait "Start" && adbax tap "Settings" --ui
+```
+
+`--ui` on any action prints the screen it led to, after waiting for the
+transition to finish. Use it whenever you need to see the result.
+
+| Want | Command |
+| --- | --- |
+| see the screen | `adbax ui` (`--all` includes layout nodes, `--full` adds classes, `--windows` adds system bars and the keyboard) |
+| find one thing | `adbax find "Delete"` (cheaper to read than a full dump) |
+| everything about a node | `adbax inspect "Delete"` (attributes plus ancestor chain) |
+| tap | `adbax tap "Sign in"` / `adbax tap 5` / `adbax tap 540,1200` |
+| type | `adbax type you@example.com --into Email --enter` |
+| replace a field's text | `adbax type "new" --into Name --clear` |
+| off-screen element | `adbax scroll-to "Delete account" --tap` |
+| block until ready | `adbax wait "Welcome" --timeout 20` (`--gone` for the inverse) |
+| hardware keys | `adbax key back home enter recents del` |
+
+Selectors match **text, content-desc and resource-id**, case-insensitively,
+exact before prefix before substring. A match that is not itself clickable
+resolves to its nearest clickable ancestor, so `adbax tap "mobile, data usage"`
+hits the whole row. `--re` switches to a regex.
+
+**Ambiguity exits 2 and lists the candidates** rather than picking one. Narrow
+the query, or pass `--nth N` / `--first`. Treat that list as the answer, not as
+an error to retry around.
+
+`adbax tap 5` reuses the last read's row numbers and skips a fresh one. Row
+numbers are only valid until the screen changes.
+
+`type` handles umlauts, CJK and emoji when the server runs (it sets the field's
+text for characters the keyboard map lacks and says so). It reads the field back
+afterwards and reports when it holds something other than what was typed.
+
+## When the tree comes back empty
+
+Compose, Views and Flutter all populate it. What does not:
+
+- Games and other canvas or GL surfaces. These draw without nodes.
+- `FLAG_SECURE` windows: payment sheets, some password managers. Screenshots of
+  them come back black as well.
+- WebViews that draw to a `<canvas>`.
+
+For those, read the screen as an image with a coordinate grid, and tap in the
+image's own pixels:
+
+```bash
+adbax screenshot --grid            # prints the path of a 540 px JPEG with a labelled grid
+adbax tap 270,480 --shot           # 270,480 in that image, scaled to the screen
+adbax swipe 100,800,440,800 --shot --dur 400
+```
+
+Read the labels off the grid rather than estimating. `--shot` always refers to
+the last screenshot taken on that device, so take a fresh one after the screen
+moves. `adbax screenshot --small` is the same image without the grid.
+
+## The server holds the accessibility connection
+
+Android allows one UiAutomation client at a time. While the adbax server runs,
+`uiautomator dump`, UiAutomator or screenshot instrumentation tests, Appium and
+Maestro cannot connect. Before running instrumentation tests, `adbax server stop`.
+It also exits on its own after ten minutes without a request.
+
+## Device and app management
+
+```bash
+adbax avds                       # list AVDs
+adbax boot Pixel_7_Pro           # boots, waits for sys.boot_completed, pins it
+adbax devices                    # * marks the pinned one
+adbax use emulator-5554          # pin when several are attached (or -s per call)
+
+adbax apps example               # installed third-party packages matching "example"
+adbax install app-debug.apk      # -r -g -t, so runtime permissions are pre-granted
+adbax install base.apk split_config.arm64_v8a.apk   # split APKs in one go
+adbax launch com.example.app     # returns once the activity is drawn
+adbax clear com.example.app      # wipe data, back to first run
+adbax deeplink "myapp://order/42"
+adbax perm grant com.example.app POST_NOTIFICATIONS
+```
+
+## Debugging
+
+```bash
+adbax clearlog && adbax launch com.example.app && adbax log com.example.app --lines 200 --level W
+adbax crash                      # the crash buffer, which survives the process dying
+adbax log com.example.app --grep "OkHttp|Retrofit"
+```
+
+`log <pkg>` resolves the current pid, so after the app restarts it shows the
+new process only. Without a package it is the whole buffer.
+
+## Screenshots worth keeping
+
+```bash
+adbax demo on                    # 10:00, full battery, full signal, no notification icons
+adbax dark on
+adbax anim off                   # also makes reads after actions settle sooner
+adbax font 1.0
+adbax screenshot ~/shots/01.png  # full resolution
+adbax demo off
+```
+
+`adbax record 20 walkthrough.mp4` for a clip.

package/src/adb.js

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: Apache-2.0
+import { execFile, execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { AdbaxError, cachePath, fail, trace } from './util.js';
+
+function sdkDir() {
+  const env = process.env.ANDROID_HOME || process.env.ANDROID_SDK_ROOT;
+  if (env) return env;
+  const home = os.homedir();
+  const usual = { darwin: 'Library/Android/sdk', win32: 'AppData/Local/Android/Sdk' }[process.platform] || 'Android/Sdk';
+  const guesses = [usual, 'Library/Android/sdk', 'Android/Sdk', 'AppData/Local/Android/Sdk'].map((g) => path.join(home, g));
+  return guesses.find((g) => fs.existsSync(g)) || guesses[0];
+}
+
+const found = {};
+/** Finds `adb` or `emulator` on PATH, then in the SDK. */
+export function binary(name, sub) {
+  if (found[name]) return found[name];
+  const exe = process.platform === 'win32' ? `${name}.exe` : name;
+  for (const dir of (process.env.PATH || '').split(path.delimiter)) {
+    const p = path.join(dir, exe);
+    if (dir && fs.existsSync(p)) return (found[name] = p);
+  }
+  const p = path.join(sdkDir(), sub, exe);
+  if (fs.existsSync(p)) return (found[name] = p);
+  fail(`${name} not found on PATH or in ${path.join(sdkDir(), sub)}. Set ANDROID_HOME.`);
+}
+
+/** Runs adb with the given argv. Resolves stdout (a string, or a Buffer with `buffer`). */
+export function adbRaw(args, { buffer = false, timeout = 120_000, soft = false } = {}) {
+  const started = performance.now();
+  return new Promise((resolve, reject) => {
+    execFile(binary('adb', 'platform-tools'), args,
+      { encoding: buffer ? 'buffer' : 'utf8', maxBuffer: 256 * 1024 * 1024, timeout },
+      (err, stdout, stderr) => {
+        trace('adb ' + args.join(' ').slice(0, 120), started);
+        if (!err) return resolve(stdout);
+        if (soft) return resolve(buffer ? Buffer.alloc(0) : '');
+        const detail = (stderr && stderr.toString().trim()) || err.message;
+        reject(new AdbaxError(`adb ${args.join(' ')}\n       ${detail}`));
+      });
+  });
+}
+
+export async function listDevices() {
+  const out = await adbRaw(['devices', '-l']);
+  return out.split('\n').slice(1).filter((l) => l.trim()).map((l) => {
+    const m = l.match(/^(\S+)\s+(\S+)(.*)$/);
+    if (!m) return null;
+    const props = {};
+    for (const kv of (m[3] || '').trim().split(/\s+/)) {
+      const i = kv.indexOf(':');
+      if (i > 0) props[kv.slice(0, i)] = kv.slice(i + 1);
+    }
+    return { serial: m[1], state: m[2], model: props.model || '', device: props.device || '' };
+  }).filter(Boolean);
+}
+
+const PIN = () => cachePath('device');
+
+export function pinnedSerial() {
+  try { return fs.readFileSync(PIN(), 'utf8').trim(); } catch { return ''; }
+}
+
+export function pinSerial(serial) {
+  if (serial) fs.writeFileSync(PIN(), serial);
+  else fs.rmSync(PIN(), { force: true });
+}
+
+/**
+ * Picks the device: -s, then $ANDROID_SERIAL, then the pinned one if it is
+ * online, then the only one online. Several and none pinned is an error rather
+ * than a guess.
+ */
+export async function resolveSerial(flags) {
+  if (flags.serial) return flags.serial;
+  if (process.env.ANDROID_SERIAL) return process.env.ANDROID_SERIAL;
+  const all = await listDevices();
+  const online = all.filter((d) => d.state === 'device');
+  const pinned = pinnedSerial();
+  if (pinned && online.some((d) => d.serial === pinned)) return pinned;
+  if (online.length === 1) return online[0].serial;
+  if (online.length === 0) {
+    const waiting = all.find((d) => d.state === 'unauthorized');
+    if (waiting) fail(`${waiting.serial} is unauthorized: accept the USB debugging prompt on the device.`);
+    if (all.length) fail(`no device online: ${all.map((d) => `${d.serial} ${d.state}`).join(', ')}.`);
+    fail('no device connected. `adbax avds` then `adbax boot <avd>`.');
+  }
+  fail(`several devices: ${online.map((d) => d.serial).join(', ')}\n` +
+    '       pick one with `adbax use <serial>` or -s <serial>.');
+}
+
+/** adb bound to one device. */
+export class Adb {
+  constructor(serial) {
+    this.serial = serial;
+  }
+
+  run(args, opts) {
+    return adbRaw(['-s', this.serial, ...args], opts);
+  }
+
+  /** `cmd` is one string for the device shell; quote user input with q(). */
+  shell(cmd, opts) {
+    return this.run(['shell', cmd], opts);
+  }
+
+  execOut(cmd, opts) {
+    return this.run(['exec-out', cmd], { buffer: true, ...opts });
+  }
+}
+
+/** The emulator binary is only needed for avds/boot, so it is looked up lazily. */
+export const emulatorBinary = () => binary('emulator', 'emulator');
+
+export function runEmulatorSync(args) {
+  return execFileSync(emulatorBinary(), args, { encoding: 'utf8' });
+}

package/src/args.js

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/** Flags that take a value: `--timeout 15` and `--timeout=15` both work. */
+const NEEDS_VALUE = new Set(['serial', 'settle', 'nth', 'timeout', 'max', 'dir', 'dur', 'in', 'into',
+  'scale', 'width', 'lines', 'grep', 'level', 'idle', 'quality']);
+/** Flags that take a value only when one follows that is not another flag. */
+const OPTIONAL_VALUE = new Set(['grid']);
+
+/**
+ * Splits argv into the command, its positional arguments and its flags.
+ * `-s <serial>` is the one short flag, kept for parity with adb. Everything
+ * after `shell` belongs to the device command, so `adbax shell logcat -s Tag`
+ * keeps its own flags.
+ */
+export function parseArgs(argv) {
+  const flags = {};
+  const pos = [];
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (pos[0] === 'shell') { pos.push(...argv.slice(i)); break; }
+    if (a === '--') { pos.push(...argv.slice(i + 1)); break; }
+    if (a.startsWith('--')) {
+      const eq = a.indexOf('=');
+      if (eq > -1) flags[a.slice(2, eq)] = a.slice(eq + 1);
+      else if (NEEDS_VALUE.has(a.slice(2)) && i + 1 < argv.length) flags[a.slice(2)] = argv[++i];
+      else if (OPTIONAL_VALUE.has(a.slice(2)) && /^\d+$/.test(argv[i + 1] || '')) flags[a.slice(2)] = argv[++i];
+      else flags[a.slice(2)] = true;
+    } else if (a === '-s' && i + 1 < argv.length) {
+      flags.serial = argv[++i];
+    } else if (a === '-h') {
+      flags.help = true;
+    } else {
+      pos.push(a);
+    }
+  }
+  return { cmd: pos.shift(), pos, flags };
+}