@ishlabs/cli 0.24.1 → 0.25.0

package/dist/lib/local-sim/android.js

@@ -0,0 +1,499 @@
+/**
+ * AndroidDevice — drives a local Android emulator/device via `adb`, implementing
+ * the SimulationDevice surface the loop expects.
+ *
+ * Two resolution paths, mirroring the browser:
+ *  - ELEMENT (preferred): observe() dumps the uiautomator a11y tree, serializes
+ *    it to the `[id] role "label"` string the backend DOMLocator reasons over,
+ *    and keeps a local `shortId → bounds` map. The backend returns a `node_id`;
+ *    executeAction() looks the bounds up locally and taps the row's CENTER.
+ *  - VISION (fallback): when the dump fails or yields a sparse tree, observe()
+ *    returns an empty tree so the backend takes its vision branch and returns
+ *    NORMALIZED 0-1000 coordinates, which we de-normalize and tap (the original
+ *    path). The vision path is also taken per-action whenever node_id is absent.
+ *
+ * Coordinate contract (see scripts/mobile-e2e + CROSS-REPO CONTRACT):
+ *   adb `screencap` and `input tap` share ONE pixel space — NO DPR correction.
+ *   - Element path: uiautomator bounds are screencap PIXELS, so the bounds-center
+ *     is already a pixel center — tap and record it as-is.
+ *   - Vision path: px = round(x / 1000 * screencapWidth); same for y.
+ */
+import { resolveTextValue } from "./actions.js";
+import { requireOneDevice, screencapPng, pngDimensions, dumpUiautomatorXml, inputTap, inputSwipe, inputDrag, inputLongPress, setUserRotation, forceStop, launchApp, installApk, isPackageInstalled, listPackages, isAdbKeyboardInstalled, enableAdbKeyboard, setIme, resetIme, currentIme, adbKeyboardType, adbKeyboardClear, pressKeyEvent, ADB_KEYBOARD_PKG, } from "./adb.js";
+import { isLocalPath } from "../upload.js";
+import { deNormalizePoint, deNormalizeDrag } from "./coordinates.js";
+import { parseUiautomatorXml, serializeNativeTree, boundsCenter } from "./native-a11y.js";
+import { packageNameFromApk } from "./apk-manifest.js";
+// Let animations/IME transitions settle before the next observation so the
+// screenshot the LLM reasons over reflects the action's result.
+const POST_GESTURE_SETTLE_MS = 500;
+async function settle(ms = POST_GESTURE_SETTLE_MS) {
+    await new Promise((r) => setTimeout(r, ms));
+}
+export class AndroidDevice {
+    contextValues;
+    log;
+    appPackage;
+    appPath;
+    /** screencap pixel size from the most recent capture — the de-normalization basis. */
+    screenWidth = 0;
+    screenHeight = 0;
+    /**
+     * shortId → bounds (screencap PIXELS) from the last observe(), the local
+     * counterpart of BrowserDevice.lastTreeData. executeAction() resolves a
+     * backend `node_id` against this and taps the bounds CENTER (element path).
+     */
+    lastNodeMap = new Map();
+    /** IME to restore on close (null if we never switched it). */
+    previousIme = null;
+    adbKeyboardActive = false;
+    constructor(opts) {
+        this.contextValues = opts.contextValues;
+        this.log = opts.log ?? (() => { });
+        this.appPackage = opts.appPackage ?? null;
+        this.appPath = opts.appPath;
+    }
+    async launchOrReset(target) {
+        await requireOneDevice();
+        // First call: install the apk (if --app is a local path) and resolve the
+        // package name to force-stop/relaunch on. `target` is the iteration's
+        // platform target (a package name) when no --app apk is supplied. Throws
+        // (rather than silently driving the foreground) if the package can't be
+        // resolved — a wrong-app run is worse than a clear error.
+        if (!this.appPackage) {
+            this.appPackage = await this.resolvePackage(target);
+        }
+        const pkg = this.appPackage;
+        // Set up ADBKeyboard once so text_input works (best-effort — text_input
+        // degrades to a no-op-with-warning if the IME isn't installed).
+        await this.ensureAdbKeyboard();
+        // Per-participant reset: stop then relaunch from a clean state.
+        await forceStop(pkg);
+        await launchApp(pkg);
+        await settle(1500); // cold start needs longer than a gesture settle
+        // Prime screencap dimensions for the first de-normalization.
+        await this.refreshDimensions();
+    }
+    /**
+     * Resolve which package to drive, returning a non-null package name or
+     * throwing. For a local .apk we read the package straight from its binary
+     * AndroidManifest (no aapt) — works whether the apk is fresh or already
+     * installed. If that parse fails we fall back to diffing the installed-package
+     * list across install, then to a foreground read; an unresolvable case throws
+     * and asks for an explicit package.
+     */
+    async resolvePackage(target) {
+        const appSpec = this.appPath ?? (target && target.trim() ? target.trim() : null);
+        if (!appSpec) {
+            throw new Error("No app to drive: pass --app <path-to.apk | installed.package.name>, or set the iteration's " +
+                "platform target to an installed package name.");
+        }
+        // `isLocalPath` returns false for http(s):// and throws on other schemes.
+        const local = isLocalPath(appSpec);
+        if (!local) {
+            // A hosted apk would need downloading to the host before `adb install`;
+            // that path isn't wired yet. Surface it loudly rather than mis-treating
+            // the URL as a package name.
+            throw new Error(`--app received a URL (${appSpec}). Installing a hosted .apk on the emulator is not supported yet — ` +
+                `pass a local .apk path or an already-installed package name.`);
+        }
+        if (appSpec.toLowerCase().endsWith(".apk")) {
+            // Read the package from the apk's manifest BEFORE install — this works
+            // even when the apk is already installed (the list-diff is empty then).
+            const manifestPkg = await packageNameFromApk(appSpec);
+            this.log(`Installing ${appSpec}...`);
+            // installApk throws on a Failure[...] result.
+            const before = manifestPkg ? null : await listPackages();
+            await installApk(appSpec);
+            if (manifestPkg) {
+                this.log(`Installed ${appSpec} → package ${manifestPkg} (from manifest)`);
+                return manifestPkg;
+            }
+            // Manifest parse failed (unusual apk layout) — fall back to the list
+            // diff, which only works on a FRESH install (the new package is the
+            // diff). We do NOT read the foreground here: launchApp hasn't run yet, so
+            // mCurrentFocus is the launcher/home — returning it would silently drive
+            // the WRONG app. When we can't tell, throw a clear error instead.
+            const after = await listPackages();
+            const added = [...after].filter((p) => !before.has(p));
+            if (added.length === 1) {
+                this.log(`Installed ${appSpec} → package ${added[0]} (from install diff)`);
+                return added[0];
+            }
+            if (added.length > 1) {
+                throw new Error(`Installing "${appSpec}" added ${added.length} packages (${added.join(", ")}); ` +
+                    `can't tell which to drive. Pass --app <package.name> explicitly.`);
+            }
+            // No new package (the apk was already installed) and no manifest — there
+            // is no reliable signal for the package. Don't guess.
+            throw new Error(`Couldn't determine the package for "${appSpec}" — it's already installed (so the install added no ` +
+                `new package) and its manifest couldn't be parsed. Pass --app <package.name> explicitly.`);
+        }
+        // Local non-.apk value: treat as an installed package name.
+        if (await isPackageInstalled(appSpec)) {
+            return appSpec;
+        }
+        throw new Error(`App package "${appSpec}" is not installed on the device and is not a local .apk path. ` +
+            `Pass --app <path-to.apk> to install it, or install it first.`);
+    }
+    async ensureAdbKeyboard() {
+        try {
+            if (!(await isAdbKeyboardInstalled())) {
+                this.log(`ADBKeyboard (${ADB_KEYBOARD_PKG}) not installed — text_input actions will be skipped. ` +
+                    `Install it on the emulator to enable typing.`);
+                return;
+            }
+            this.previousIme = await currentIme();
+            await enableAdbKeyboard();
+            this.adbKeyboardActive = true;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            this.log(`Could not activate ADBKeyboard — text_input will be skipped: ${msg}`);
+        }
+    }
+    async refreshDimensions() {
+        const png = await screencapPng();
+        const { width, height } = pngDimensions(png);
+        this.screenWidth = width;
+        this.screenHeight = height;
+        return png;
+    }
+    async observe() {
+        // Screencap and the a11y dump are independent reads — run them in parallel.
+        // The dump is wrapped so a failure degrades to the vision path (empty tree)
+        // rather than aborting the observation.
+        const [png, tree] = await Promise.all([
+            this.refreshDimensions(),
+            this.dumpTree(),
+        ]);
+        this.lastNodeMap = tree.nodeMap;
+        return {
+            screenshot: png.toString("base64"),
+            // Element path when the dump produced a tree; "" → backend vision branch.
+            accessibilityTree: tree.simplified,
+            url: "",
+            width: this.screenWidth,
+            height: this.screenHeight,
+            // Native has no scrollable document; the screen IS the page.
+            documentHeight: this.screenHeight,
+            tabs: [],
+        };
+    }
+    /**
+     * Dump + serialize the uiautomator a11y tree. Any failure (dump retries
+     * exhausted, parse error) degrades to an empty tree so the backend falls back
+     * to the vision path — a missing tree must never abort the observation.
+     */
+    async dumpTree() {
+        try {
+            const xml = await dumpUiautomatorXml();
+            const nodes = parseUiautomatorXml(xml);
+            const tree = serializeNativeTree(nodes);
+            this.log(`a11y tree: ${tree.nodeMap.size} node(s)`);
+            return tree;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            this.log(`a11y dump failed, falling back to vision: ${msg}`);
+            return { simplified: "", nodeMap: new Map() };
+        }
+    }
+    async captureScreenshot() {
+        const png = await this.refreshDimensions();
+        return png.toString("base64");
+    }
+    async captureScreenshotJpeg() {
+        // adb screencap only emits PNG. We return the PNG bytes; the upload/record
+        // path treats them as opaque image bytes (PDQ frame-matching works on PNG).
+        return this.refreshDimensions();
+    }
+    dimensions() {
+        return { width: this.screenWidth, height: this.screenHeight };
+    }
+    /** Normalized 0-1000 → screencap pixel space. NO DPR correction. */
+    toPixels(c) {
+        return deNormalizePoint(c, this.screenWidth, this.screenHeight);
+    }
+    /**
+     * Resolve the pixel target for a positional action. The ELEMENT path wins when
+     * the backend returned a `node_id`: look the bounds up in the last observe()'s
+     * nodeMap and tap the row's CENTER (already in screencap pixels). Otherwise the
+     * VISION path de-normalizes the backend's 0-1000 coordinates. Returns:
+     *  - {target} on success,
+     *  - {stale:true} when a node_id has no bounds (the tree moved under us) — the
+     *    caller fails the action so the loop forwards DOM_ELEMENT_NOT_FOUND and the
+     *    agent re-observes/retries,
+     *  - {target:null} when neither node_id nor coordinates were supplied.
+     */
+    resolveTarget(action) {
+        if (action.node_id) {
+            const bounds = this.lastNodeMap.get(action.node_id);
+            if (!bounds)
+                return { target: null, stale: true };
+            return { target: boundsCenter(bounds), stale: false };
+        }
+        return { target: action.coordinates ? this.toPixels(action.coordinates) : null, stale: false };
+    }
+    async executeAction(action) {
+        try {
+            // ELEMENT path (node_id → bounds center, screencap pixels) or VISION path
+            // (0-1000 → pixels). A stale node_id fails like an unresolved target.
+            const resolved = this.resolveTarget(action);
+            if (resolved.stale)
+                return this.failStaleNode(action);
+            const px = resolved.target;
+            switch (action.type) {
+                case "tap":
+                case "double_tap": {
+                    if (!px)
+                        return this.failNoCoords(action);
+                    const count = action.type === "double_tap" ? 2 : action.count ?? 1;
+                    for (let i = 0; i < count; i++)
+                        await inputTap(px.x, px.y);
+                    break;
+                }
+                case "long_press": {
+                    if (!px)
+                        return this.failNoCoords(action);
+                    await inputLongPress(px.x, px.y, action.duration_ms ?? 600);
+                    break;
+                }
+                case "text_input": {
+                    await this.typeText(action, px);
+                    break;
+                }
+                case "scroll": {
+                    await this.scroll(action);
+                    break;
+                }
+                case "swipe":
+                case "pull_to_refresh": {
+                    await this.swipe(action.direction ?? (action.type === "pull_to_refresh" ? "down" : "up"));
+                    break;
+                }
+                case "navigate_back": {
+                    await pressKeyEvent("KEYCODE_BACK");
+                    break;
+                }
+                case "drag": {
+                    // A drag GRABS an element and RELEASES it elsewhere ("click the
+                    // element, move, let go") — distinct from a swipe (element-less
+                    // directional). Press the resolved element center (px — the same
+                    // element path a tap uses), move to the drop point, release. Record
+                    // the grab point as the action's coordinates.
+                    const start = await this.drag(action, px);
+                    if (!start)
+                        return this.failNoCoords(action);
+                    await settle();
+                    return {
+                        success: true,
+                        elementName: action.element_name,
+                        coordinates: start,
+                        openedNewTab: false,
+                    };
+                }
+                case "rotate_device": {
+                    await this.rotate(action.orientation === "landscape" ? "landscape" : "portrait");
+                    break;
+                }
+                case "wait": {
+                    await settle(action.duration_ms ?? 1000);
+                    break;
+                }
+                case "think": {
+                    // Reasoning-only: no device interaction.
+                    return { success: true, elementName: action.element_name, coordinates: null, openedNewTab: false };
+                }
+                case "pinch_zoom":
+                case "keyboard_shortcut":
+                case "switch_tab":
+                case "close_tab": {
+                    // True multi-touch / browser-tab / desktop concepts with no faithful
+                    // single-app native equivalent — fail LOUDLY (not silently) so the
+                    // loop forwards it and the agent can adapt (e.g. double_tap to zoom).
+                    return this.failUnsupported(action);
+                }
+                default: {
+                    this.log(`Unknown native action: ${action.type}`);
+                    return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+                }
+            }
+            await settle();
+            return {
+                success: true,
+                elementName: action.element_name,
+                // Report the pixel target so the loop re-normalizes it for recording
+                // (element path: the bounds-center; vision path: the de-normalized
+                // 0-1000 → round-trips exactly against dimensions(), screencap pixels).
+                coordinates: px,
+                openedNewTab: false,
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            this.log(`Action ${action.type} failed: ${msg}`);
+            return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+        }
+    }
+    failNoCoords(action) {
+        // The backend couldn't vision-locate a target (coordinates=null). Skip the
+        // action (don't crash, don't silently succeed) and surface it like the
+        // browser path's unresolved-element case — success:false makes the loop
+        // push a DOM_ELEMENT_NOT_FOUND forward so the LLM learns the target missed.
+        const target = action.element_description || action.element_name || "(no description)";
+        this.log(`Skipping native action with no resolved target: ${action.type} ${target}`);
+        return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+    }
+    failStaleNode(action) {
+        // The backend resolved a node_id that's no longer in the latest tree (the
+        // screen changed between observe() and act). Fail like the browser's
+        // unresolved-element case — success:false forwards DOM_ELEMENT_NOT_FOUND so
+        // the loop re-observes and the agent retries against a fresh tree.
+        const target = action.element_description || action.element_name || action.node_id || "(unknown)";
+        this.log(`Stale node_id "${action.node_id}" not in current a11y tree: ${action.type} ${target}`);
+        return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+    }
+    async typeText(action, px) {
+        if (!this.adbKeyboardActive) {
+            this.log("text_input skipped: ADBKeyboard not active.");
+            return;
+        }
+        // Focus the field first if the model gave a target.
+        if (px) {
+            await inputTap(px.x, px.y);
+            await settle(250);
+        }
+        const text = resolveTextValue(action, this.contextValues);
+        if (action.mode === "click_type") {
+            await adbKeyboardClear();
+        }
+        await adbKeyboardType(text);
+        if (action.submit) {
+            await settle(150);
+            await pressKeyEvent("KEYCODE_ENTER");
+        }
+    }
+    async scroll(action) {
+        const w = this.screenWidth;
+        const h = this.screenHeight;
+        const cx = Math.round(w / 2);
+        const amountMap = {
+            small: 0.25, medium: 0.45, large: 0.7, extra_large: 0.9,
+        };
+        const frac = amountMap[action.amount ?? "medium"] ?? 0.45;
+        const dist = Math.round(h * frac);
+        const mid = Math.round(h / 2);
+        switch (action.direction) {
+            case "up":
+                // Reveal content above: swipe finger downward.
+                await inputSwipe(cx, mid - dist / 2, cx, mid + dist / 2);
+                break;
+            case "to_top":
+                await inputSwipe(cx, Math.round(h * 0.2), cx, Math.round(h * 0.9), 400);
+                break;
+            case "to_bottom":
+                await inputSwipe(cx, Math.round(h * 0.9), cx, Math.round(h * 0.2), 400);
+                break;
+            case "down":
+            case "to_element":
+            default:
+                // Reveal content below: swipe finger upward.
+                await inputSwipe(cx, mid + dist / 2, cx, mid - dist / 2);
+                break;
+        }
+    }
+    async swipe(direction) {
+        const w = this.screenWidth;
+        const h = this.screenHeight;
+        const cx = Math.round(w / 2);
+        const cy = Math.round(h / 2);
+        const d = Math.round(h * 0.4);
+        const dx = Math.round(w * 0.4);
+        switch (direction) {
+            case "up":
+                await inputSwipe(cx, cy + d / 2, cx, cy - d / 2);
+                break;
+            case "down":
+                await inputSwipe(cx, cy - d / 2, cx, cy + d / 2);
+                break;
+            case "left":
+                await inputSwipe(cx + dx / 2, cy, cx - dx / 2, cy);
+                break;
+            case "right":
+                await inputSwipe(cx - dx / 2, cy, cx + dx / 2, cy);
+                break;
+            default:
+                await inputSwipe(cx, cy + d / 2, cx, cy - d / 2);
+                break;
+        }
+    }
+    /**
+     * Perform a drag: press the GRABBED element, move to the drop point, release.
+     * A drag is "click an element and let it go", so the press lands element-
+     * center (the resolved `grab` — node_id bounds center, or the vision
+     * coordinate when the tree is blind), NOT the backend's vision-estimated
+     * start. The release point is the drag END (drag.endX/endY). Both the grab
+     * fallback and the end de-normalize against screencap pixels. `inputDrag`
+     * (`input draganddrop`) dwells at the press point first so a long-press
+     * pickup registers — a slow swipe would read as a directional swipe instead.
+     * Returns the grab pixel point to record, or null if there's no end to drag
+     * toward.
+     */
+    async drag(action, grab) {
+        if (!action.drag)
+            return null;
+        const { start, end } = deNormalizeDrag(action.drag, this.screenWidth, this.screenHeight);
+        // Grab the resolved element center; fall back to the backend's own start
+        // only when nothing resolved (no node_id and no vision coordinate).
+        const press = grab ?? start;
+        await inputDrag(press.x, press.y, end.x, end.y);
+        return press;
+    }
+    async rotate(orientation) {
+        await setUserRotation(orientation);
+        // Orientation changes the screencap geometry — re-read so the next
+        // de-normalization uses the rotated dimensions.
+        await settle();
+        await this.refreshDimensions();
+    }
+    failUnsupported(action) {
+        // A capability the single-app native driver genuinely can't perform (true
+        // multi-touch pinch, browser tabs, desktop keyboard shortcuts). Fail with a
+        // clear, diagnosable reason — NOT a silent false — so it's visible WHY and
+        // the agent can adapt.
+        const hint = {
+            pinch_zoom: "no multi-touch on the native driver; the agent should zoom via double_tap",
+            keyboard_shortcut: "no hardware keyboard on the native driver; use on-screen taps/text_input",
+            switch_tab: "tabs are a browser concept; the native app has a single window",
+            close_tab: "tabs are a browser concept; the native app has a single window",
+        };
+        this.log(`${action.type} not supported by the Android native driver — ${hint[action.type] ?? "no native equivalent"}`);
+        return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+    }
+    currentUrl() {
+        // Native has no URL; recording stores "" (current_location comes from the
+        // backend's reasoning output, not the device).
+        return "";
+    }
+    async close() {
+        // Restore the IME if we switched it. If we had a prior IME, set it back;
+        // if there wasn't one we could read (previousIme null), at least take the
+        // shared emulator off ADBKeyboard so we don't leave it on our test IME.
+        if (this.adbKeyboardActive) {
+            try {
+                if (this.previousIme) {
+                    await setIme(this.previousIme);
+                }
+                else {
+                    await resetIme();
+                }
+            }
+            catch {
+                // Non-fatal — don't crash teardown over IME state on a shared device.
+            }
+        }
+        // Leave the app installed/running; the emulator is shared and the next run
+        // resets via launchOrReset. Nothing else to tear down.
+    }
+}

package/dist/lib/local-sim/apk-manifest.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Extract an APK's package name from its binary AndroidManifest.xml — pure,
+ * dependency-free (no aapt, no SDK), so `--app <apk>` resolves the package even
+ * when the apk is ALREADY installed (the install-list diff is empty then).
+ *
+ * Two steps, both on in-memory bytes:
+ *  1. Pull AndroidManifest.xml out of the APK (a ZIP) — find its local file
+ *     header and inflate (raw deflate) or copy (stored).
+ *  2. Parse the binary XML (AXML): read the string pool, find the <manifest>
+ *     START_ELEMENT, and read its `package` attribute's string value.
+ *
+ * We only need the package string, so this is a deliberately minimal AXML
+ * reader (not a general decoder). Returns null on anything unexpected — the
+ * caller falls back to other resolution and never crashes on a weird apk.
+ */
+/**
+ * Parse binary AXML and return the <manifest> element's `package` attribute.
+ * Returns null if the structure isn't what we expect.
+ */
+export declare function parseAxmlPackage(axml: Buffer): string | null;
+/** Read an APK file and return its package name, or null if it can't be parsed. */
+export declare function packageNameFromApk(apkPath: string): Promise<string | null>;