@ishlabs/cli 0.24.0 → 0.25.0

package/dist/lib/local-sim/native-a11y.js

@@ -0,0 +1,384 @@
+/**
+ * Pure parser/serializer for native (Android/iOS) accessibility trees — the
+ * native counterpart of the browser's DOM-locator tree. It turns a raw device
+ * a11y dump into the SAME `[id] role "name"` string the backend's DOMLocator
+ * reasons over, plus a local `shortId → bounds` map the device taps the CENTER
+ * of. No bounds ship to the backend; like the browser path, the CLI keeps the
+ * map and resolves the LLM's returned short id locally.
+ *
+ * FCIS: this module is pure (string in, structs out) — no `adb`/`idb` I/O — so
+ * it's unit-testable without a device, exactly like `coordinates.ts`. The I/O
+ * lives in `adb.ts`/`simctl.ts`; the parse/serialize math lives here.
+ *
+ * COORDINATE SPACE — carried, not converted, by this module:
+ *   - Android `uiautomator dump` bounds are screencap PIXELS (`space: "px"`).
+ *   - iOS `idb ui describe-all` frames are POINTS (`space: "points"`).
+ * The device de-normalizes/taps in its own space (AndroidDevice taps pixels;
+ * IOSDevice taps points), so the `space` tag tells the caller which dimension a
+ * node's bounds-center belongs to. This module never mixes the two.
+ *
+ * ANCESTOR-VS-LEAF (the hard part): on Android the visible label
+ * ("Network & internet") sits on a `clickable=false` TextView nested inside the
+ * clickable PARENT row. Tapping the leaf's center misses the row's hit logic and
+ * lands "slightly off"; the click target is the row. So the serializer walks to
+ * the nearest clickable ANCESTOR, aggregates its descendants' text/content-desc
+ * into ONE label, and emits the CLICKABLE node WITH THE ROW'S BOUNDS — never the
+ * leaf. iOS Buttons are already labeled + actionable, so they emit directly.
+ */
+/**
+ * Native role → ARIA synonym so the DOMLocator prompt sees the same vocabulary
+ * it does for the web (Option A from the Phase-0 spike). One small lookup so the
+ * map is trivial to tweak once the spike finalizes it; unknown roles pass
+ * through lower-cased (better a literal native role than a dropped node).
+ */
+const ROLE_NORMALIZATION = {
+    // Android (android.widget.* / android.view.*, matched on the leaf class name).
+    Button: "button",
+    ImageButton: "button",
+    EditText: "textbox",
+    AutoCompleteTextView: "textbox",
+    TextView: "text",
+    CheckBox: "checkbox",
+    RadioButton: "radio",
+    Switch: "switch",
+    ToggleButton: "switch",
+    ImageView: "image",
+    ViewGroup: "generic",
+    LinearLayout: "generic",
+    FrameLayout: "generic",
+    RelativeLayout: "generic",
+    ScrollView: "generic",
+    RecyclerView: "list",
+    ListView: "list",
+    // iOS (idb `type`, AX-prefixed `role` handled by stripAxPrefix below).
+    StaticText: "text",
+    TextField: "textbox",
+    SecureTextField: "textbox",
+    SearchField: "searchbox",
+    Cell: "listitem",
+    Heading: "heading",
+    Image: "image",
+    Group: "generic",
+    Link: "link",
+    Application: "application",
+};
+/** Roles that are pure layout containers — only kept if they're a tappable row. */
+const DECORATIVE_GENERIC_ROLES = new Set(["generic", "application"]);
+function normalizeRole(rawRole) {
+    const key = stripAxPrefix(rawRole);
+    return ROLE_NORMALIZATION[key] ?? key.toLowerCase();
+}
+/** "AXButton" → "Button", "AXStaticText" → "StaticText"; non-AX passes through. */
+function stripAxPrefix(role) {
+    return role.startsWith("AX") ? role.slice(2) : role;
+}
+const ANDROID_BOUNDS_RE = /^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$/;
+/** Parse `bounds="[x1,y1][x2,y2]"` → Bounds, or null if malformed/zero-area. */
+function parseAndroidBounds(raw) {
+    const m = ANDROID_BOUNDS_RE.exec(raw.trim());
+    if (!m)
+        return null;
+    const x1 = Number(m[1]);
+    const y1 = Number(m[2]);
+    const x2 = Number(m[3]);
+    const y2 = Number(m[4]);
+    const width = x2 - x1;
+    const height = y2 - y1;
+    // A zero-area rect ([0,0][0,0], collapsed/off-screen rows) has no tappable
+    // center — treat it as malformed so it's dropped, not tapped at (0,0).
+    if (width <= 0 || height <= 0)
+        return null;
+    return { x: x1, y: y1, width, height };
+}
+/** Last segment of a dotted class, e.g. "android.widget.TextView" → "TextView". */
+function androidLeafClass(cls) {
+    const dot = cls.lastIndexOf(".");
+    return dot >= 0 ? cls.slice(dot + 1) : cls;
+}
+function attr(tag, name) {
+    // uiautomator attributes are double-quoted with XML entity escapes. Match the
+    // literal attr and unescape the handful of entities uiautomator emits.
+    const m = new RegExp(`\\s${name}="([^"]*)"`).exec(tag);
+    return m ? unescapeXml(m[1]) : "";
+}
+function unescapeXml(s) {
+    return s
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'");
+}
+/**
+ * Parse a uiautomator XML dump into a flat list of leaf-significant nodes in
+ * document order. The dump is a single line of nested `<node ...>` tags; we
+ * rebuild the parent/child nesting from the open/close-tag stream (mirroring the
+ * "break after `>`" split the oracle scripts use, but tracking depth so the
+ * ancestor-aggregation in `serializeNativeTree` has the real tree).
+ *
+ * Returns the FLATTENED set of nodes (depth-first, document order) with their
+ * raw fields; the serializer decides which to emit and how to aggregate.
+ */
+export function parseUiautomatorXml(xml) {
+    const root = buildAndroidTree(xml);
+    const out = [];
+    const visit = (n) => {
+        // Drop nodes with no usable bounds (malformed/zero-area) — they have no
+        // tappable center and would corrupt the nodeMap.
+        if (n.bounds) {
+            const label = n.text || n.contentDesc;
+            out.push({
+                role: normalizeRole(androidLeafClass(n.role)),
+                label,
+                bounds: n.bounds,
+                clickable: n.clickable,
+                hasOwnLabel: label.length > 0,
+                resourceId: n.resourceId || undefined,
+                space: "px",
+            });
+        }
+        for (const c of n.children)
+            visit(c);
+    };
+    for (const c of root.children)
+        visit(c);
+    return out;
+}
+/**
+ * Rebuild the uiautomator node tree from the flat tag stream. uiautomator emits
+ * `<node ...>...</node>` (container) and `<node .../>` (self-closing leaf) on one
+ * line; we tokenize the tags and use an explicit stack so each node's children
+ * are its true descendants — required for ancestor-vs-leaf aggregation.
+ */
+function buildAndroidTree(xml) {
+    const root = makeRawAndroidNode("", "", "", "", false, null);
+    const stack = [root];
+    // Match every <node ...> / <node .../> open tag and standalone </node> close.
+    // Attribute values are consumed as atomic quoted runs (`"[^"]*"`) so a literal
+    // `>` INSIDE a value can't terminate the tag early — uiautomator escapes `&`,
+    // `<`, `"` but NOT `>`, so a label like text="Home > Settings" (breadcrumbs)
+    // would otherwise truncate the tag and drop the whole node.
+    const tagRe = /<node\b(?:[^>"]|"[^"]*")*>|<\/node>/g;
+    let m;
+    while ((m = tagRe.exec(xml)) !== null) {
+        const tag = m[0];
+        if (tag === "</node>") {
+            if (stack.length > 1)
+                stack.pop();
+            continue;
+        }
+        // Self-closing is read off the matched tag (`.../>`), not a capture group:
+        // the greedy run above swallows the trailing slash, so a `(\/?)` capture
+        // can't see it.
+        const selfClosing = tag.endsWith("/>");
+        const node = makeRawAndroidNode(attr(tag, "class"), attr(tag, "text"), attr(tag, "content-desc"), attr(tag, "resource-id"), attr(tag, "clickable") === "true", parseAndroidBounds(attr(tag, "bounds")));
+        stack[stack.length - 1].children.push(node);
+        if (!selfClosing)
+            stack.push(node);
+    }
+    return root;
+}
+function makeRawAndroidNode(role, text, contentDesc, resourceId, clickable, bounds) {
+    return { role, text, contentDesc, resourceId, clickable, bounds, children: [] };
+}
+/** iOS roles/types that are directly actionable (the device taps their center). */
+const IOS_ACTIONABLE_TYPES = new Set([
+    "Button",
+    "Link",
+    "TextField",
+    "SecureTextField",
+    "SearchField",
+    "Cell",
+    "Switch",
+    "Checkbox",
+    "RadioButton",
+    "MenuItem",
+    "Tab",
+]);
+/**
+ * Parse `idb ui describe-all` JSON (a FLAT array of elements, each with a `frame`
+ * in POINTS) into NativeNodes in array order. iOS is already a flat,
+ * properly-labeled list — no ancestor walk needed — so `clickable` is derived
+ * from the element's role/type and whether it carries a usable label.
+ */
+export function parseIdbDescribeAll(json) {
+    let parsed;
+    try {
+        parsed = JSON.parse(json);
+    }
+    catch {
+        return [];
+    }
+    if (!Array.isArray(parsed))
+        return [];
+    const out = [];
+    for (const raw of parsed) {
+        const bounds = idbFrameToBounds(raw.frame);
+        if (!bounds)
+            continue; // malformed / zero-area frame → no tappable center
+        // Label: prefer the spoken AXLabel; fall back to AXValue (search fields
+        // expose their placeholder as AXValue, e.g. "Search"). AXValue is only a
+        // STRING fallback — switches/sliders/steppers report it as a number/boolean
+        // (a Switch is 1/0), and `.trim()` on those would throw and lose the whole
+        // tree to a silent vision fallback. An unlabeled toggle then emits as a bare
+        // `[id] switch` (still tappable via its frame center).
+        const label = (raw.AXLabel ?? (typeof raw.AXValue === "string" ? raw.AXValue : "")).trim();
+        const rawType = raw.type ?? (raw.role ? stripAxPrefix(raw.role) : "");
+        const typeKey = stripAxPrefix(rawType);
+        const actionable = IOS_ACTIONABLE_TYPES.has(typeKey) && raw.enabled !== false;
+        out.push({
+            role: normalizeRole(rawType),
+            label,
+            bounds,
+            clickable: actionable,
+            hasOwnLabel: label.length > 0,
+            resourceId: raw.AXUniqueId ?? undefined,
+            space: "points",
+        });
+    }
+    return out;
+}
+function idbFrameToBounds(frame) {
+    if (!frame)
+        return null;
+    const { x, y, width, height } = frame;
+    if (typeof x !== "number" ||
+        typeof y !== "number" ||
+        typeof width !== "number" ||
+        typeof height !== "number" ||
+        !Number.isFinite(x) ||
+        !Number.isFinite(y) ||
+        width <= 0 ||
+        height <= 0) {
+        return null;
+    }
+    return { x, y, width, height };
+}
+// ---------------------------------------------------------------------------
+// Serialization — flat NativeNode list → `[id] role "label"` + nodeMap
+// ---------------------------------------------------------------------------
+const NODE_LABEL_MAX_LENGTH = 100;
+/** Stable short-id prefix per space so a mixed log is unambiguous. */
+function shortIdPrefix(space) {
+    return space === "px" ? "A" : "I";
+}
+function truncate(text, max) {
+    return text.length <= max ? text : text.slice(0, max - 1) + "…";
+}
+/** Collapse runs of whitespace so aggregated multi-line labels read on one line. */
+function normalizeLabel(label) {
+    return label.replace(/\s+/g, " ").trim();
+}
+/**
+ * Serialize a flat NativeNode list (from `parseUiautomatorXml` /
+ * `parseIdbDescribeAll`) into the `[id] role "label"` string the DOMLocator
+ * reasons over, plus a `shortId → bounds` map for local tap resolution.
+ *
+ * Emission rules (kept tight, like the DOM serializer):
+ *  - ANCESTOR-VS-LEAF: a CLICKABLE node absorbs its descendants' labels and is
+ *    emitted with ITS OWN bounds (the tappable row). The descendant
+ *    label-bearing leaves are then NOT emitted on their own — their text lives
+ *    on the row. A label-bearing leaf with NO clickable ancestor (e.g. a
+ *    standalone heading) is emitted directly so on-screen text isn't lost.
+ *  - Skip pure decoration: a node that is neither clickable nor label-bearing,
+ *    and a generic/application container that didn't aggregate a label.
+ *
+ * The input list is depth-first / document order, which is the order the raw
+ * parsers produce; we recover ancestry from that order using bounds containment
+ * (Android leaves nest inside their clickable row's rect; iOS is already flat).
+ */
+export function serializeNativeTree(nodes) {
+    const space = nodes[0]?.space ?? "px";
+    const prefix = shortIdPrefix(space);
+    // 1) Aggregate descendant labels onto their nearest clickable ancestor. A
+    //    descendant is a label-bearing node whose bounds sit inside a clickable
+    //    node's bounds (and is not itself the clickable node). Document order
+    //    guarantees an ancestor appears before its descendants on Android.
+    const clickables = nodes.filter((n) => n.clickable);
+    const aggregatedLabel = new Map();
+    const consumedAsDescendant = new Set();
+    for (const node of nodes) {
+        if (!node.hasOwnLabel)
+            continue;
+        // Find the smallest clickable rect that contains this label node. Skip the
+        // node itself (a clickable node keeps its own label).
+        const host = smallestContainingClickable(node, clickables);
+        if (host && host !== node) {
+            const bucket = aggregatedLabel.get(host) ?? [];
+            bucket.push(node.label);
+            aggregatedLabel.set(host, bucket);
+            consumedAsDescendant.add(node);
+        }
+    }
+    // 2) Emit. Walk in document order so ids follow the on-screen reading order.
+    const lines = [];
+    const nodeMap = new Map();
+    let counter = 0;
+    for (const node of nodes) {
+        if (consumedAsDescendant.has(node))
+            continue; // its label moved to the row
+        const aggregated = aggregatedLabel.get(node);
+        const ownLabel = node.label;
+        // A clickable node's label = its own label + any absorbed descendant labels;
+        // its own label leads (iOS Buttons carry it directly).
+        const combined = normalizeLabel([ownLabel, ...(aggregated ?? [])].filter(Boolean).join(" "));
+        const emit = shouldEmit(node, combined);
+        if (!emit)
+            continue;
+        counter += 1;
+        const shortId = `${prefix}${counter}`;
+        const label = truncate(combined, NODE_LABEL_MAX_LENGTH);
+        const labelPart = label ? ` "${label}"` : "";
+        lines.push(`[${shortId}] ${node.role}${labelPart}`);
+        nodeMap.set(shortId, node.bounds);
+    }
+    return { simplified: lines.join("\n"), nodeMap };
+}
+/**
+ * Decide whether a node makes the simplified tree. Keep it if it's actionable,
+ * or it carries (aggregated) text and isn't a bare layout container. Drop pure
+ * decoration: unlabeled non-clickable nodes, and generic/application containers
+ * that absorbed no label.
+ */
+function shouldEmit(node, combinedLabel) {
+    if (node.clickable)
+        return true;
+    if (!combinedLabel)
+        return false;
+    // A labeled non-clickable node is real on-screen text (heading, value);
+    // keep it unless it's a bare generic container with no useful role signal.
+    return !DECORATIVE_GENERIC_ROLES.has(node.role);
+}
+/**
+ * Return the smallest-area clickable node whose bounds CONTAIN `node`'s bounds
+ * (its center inside the rect), or null. "Smallest" picks the immediate row, not
+ * a giant scroll container wrapping everything.
+ */
+function smallestContainingClickable(node, clickables) {
+    const cx = node.bounds.x + node.bounds.width / 2;
+    const cy = node.bounds.y + node.bounds.height / 2;
+    let best = null;
+    let bestArea = Infinity;
+    for (const c of clickables) {
+        if (c === node) {
+            best = c; // a clickable node hosts its own label
+            bestArea = c.bounds.width * c.bounds.height;
+            continue;
+        }
+        if (!containsPoint(c.bounds, cx, cy))
+            continue;
+        const area = c.bounds.width * c.bounds.height;
+        if (area < bestArea) {
+            best = c;
+            bestArea = area;
+        }
+    }
+    return best;
+}
+function containsPoint(b, x, y) {
+    return x >= b.x && x <= b.x + b.width && y >= b.y && y <= b.y + b.height;
+}
+/** Center of a node's bounds — the point the device taps. */
+export function boundsCenter(b) {
+    return { x: Math.round(b.x + b.width / 2), y: Math.round(b.y + b.height / 2) };
+}

package/dist/lib/local-sim/simctl.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Thin async wrappers over `xcrun simctl` + `idb` for the native-iOS sim path.
+ *
+ * Two tools, two jobs:
+ *  - `xcrun simctl` drives the simulator LIFECYCLE (boot detection, install,
+ *    terminate, launch) and the SCREENSHOT.
+ *  - `idb` drives UI INPUT (tap/swipe/text/key) and reports the screen
+ *    geometry (pixels, points, and the scale between them).
+ *
+ * COORDINATE SPACES (the key difference from Android, where screencap and tap
+ * share one pixel space):
+ *  - `simctl io booted screenshot` writes a PNG in PIXELS (e.g. 1179x2556 @3x).
+ *  - `idb ui tap/swipe` take POINTS (e.g. 393x852) — pixels / scale.
+ * The native sim TAPS in points (de-normalize 0-1000 against the POINT size)
+ * but RECORDS in PIXELS: dimensions() returns the pixel size so the loop's
+ * round-trip is exact. Recording in points would drift — the point grid (393)
+ * is coarser than the 0-1000 normalized grid, so it double-rounds. See
+ * IOSDevice for the full derivation.
+ */
+export declare class IosError extends Error {
+    constructor(message: string);
+}
+/** Run `xcrun simctl <args>` and return trimmed stdout. */
+export declare function simctl(args: string[], timeoutMs?: number): Promise<string>;
+/** Run `idb <args>` and return trimmed stdout. */
+export declare function idb(args: string[], timeoutMs?: number): Promise<string>;
+/**
+ * Assert exactly one simulator is Booted and return its udid. We pin every
+ * subsequent idb/simctl call (and the screenshot) to "booted", so multiple
+ * booted simulators are ambiguous and rejected.
+ */
+export declare function requireOneBootedSimulator(): Promise<string>;
+/**
+ * Screen geometry from `idb describe --json`: PIXEL size, POINT size, and the
+ * scale (`density`) between them. Points drive idb ui tap/swipe; pixels are the
+ * screenshot's resolution.
+ */
+export interface IosScreen {
+    pixelWidth: number;
+    pixelHeight: number;
+    pointWidth: number;
+    pointHeight: number;
+    density: number;
+}
+export declare function describeScreen(udid: string): Promise<IosScreen>;
+/**
+ * Capture the booted simulator's screen as PNG bytes via
+ * `simctl io booted screenshot`. simctl writes to a file path (no reliable
+ * stdout in current Xcode), so we round-trip through a temp file.
+ */
+export declare function screenshotPng(): Promise<Buffer>;
+export declare function uiTap(udid: string, x: number, y: number): Promise<void>;
+export declare function uiLongPress(udid: string, x: number, y: number, durationMs?: number): Promise<void>;
+export declare function uiSwipe(udid: string, x1: number, y1: number, x2: number, y2: number, durationMs?: number): Promise<void>;
+/**
+ * Type text into the focused field. Unlike Android's `adb shell input text`,
+ * `idb ui text` handles spaces/unicode/quotes correctly, so no helper IME is
+ * needed.
+ */
+export declare function uiText(udid: string, text: string): Promise<void>;
+/**
+ * Press a hardware key by HID usage code. `idb ui key 40` is Return/Enter
+ * (used to submit a text field).
+ */
+export declare function uiKey(udid: string, keycode: number): Promise<void>;
+/** HID usage code for Return/Enter. */
+export declare const HID_KEY_RETURN = 40;
+/**
+ * Capture the current accessibility tree as `idb ui describe-all` JSON (a flat
+ * array of elements, each with a POINT frame) and return it. Mirrors the
+ * oracle's `ios_describe`: right after a tap the tree can be mid-transition and
+ * come back empty/partial, so we retry until we get an array with more than just
+ * the root application node. Throws IosError if every attempt yields a trivial
+ * tree so the caller can degrade to the vision path.
+ */
+export declare function describeAll(udid: string): Promise<string>;
+export declare function terminateApp(udid: string, bundleId: string): Promise<void>;
+export declare function launchApp(udid: string, bundleId: string): Promise<void>;
+export declare function installApp(udid: string, appPath: string): Promise<void>;
+export declare function isAppInstalled(udid: string, bundleId: string): Promise<boolean>;
+/**
+ * Read CFBundleIdentifier from a local `.app`'s Info.plist via `plutil`. Lets us
+ * terminate+launch a just-installed app without diffing the app list.
+ */
+export declare function bundleIdFromApp(appPath: string): Promise<string | null>;