@ishlabs/cli 0.24.1 → 0.26.0

package/dist/lib/local-sim/ios.js

@@ -0,0 +1,599 @@
+/**
+ * IOSDevice — drives a local iOS simulator via `xcrun simctl` (lifecycle +
+ * screenshot) and WebDriverAgent/XCUITest (UI + a11y; see xcuitest.ts),
+ * implementing the SimulationDevice surface the loop expects. Mirrors
+ * AndroidDevice; the one substantive difference is the coordinate space.
+ *
+ * Two resolution paths, mirroring the browser:
+ *  - ELEMENT (preferred): observe() reads WDA's `/source` a11y tree, serializes
+ *    it to the `[id] role "label"` string the backend DOMLocator reasons over,
+ *    and keeps a local `shortId → bounds` map (bounds in POINTS). The backend
+ *    returns a `node_id`; executeAction() looks the bounds up and taps the
+ *    element's CENTER.
+ *  - VISION (fallback): when the tree is empty/sparse, observe() returns an
+ *    empty tree so the backend takes its vision branch and returns NORMALIZED
+ *    0-1000 coordinates. Also taken per-action whenever node_id is absent.
+ *
+ * COORDINATE SPACE — two spaces, the key difference from Android (where
+ * screencap and tap share one pixel space):
+ *   `simctl io booted screenshot` is in PIXELS (e.g. 1179x2556 @3x), but
+ *   WDA taps/swipes AND the `/source` a11y frames are POINTS (393x852).
+ *   The invariant in BOTH paths: TAP in points, RECORD in pixels, because the
+ *   loop re-normalizes the recorded coord against dimensions() (PIXELS).
+ *     - VISION:  tap pt = round(n/1000 * pointSize); record px = round(n/1000 * pixelSize).
+ *     - ELEMENT: tap = bounds-center (already POINTS); record = that center
+ *       scaled POINTS→PIXELS via pointToPixel() (the @Nx scale).
+ *   dimensions() returns the PIXEL size, so the loop re-normalizes the recorded
+ *   px back to a stable 0-1000. Recording in points would drift: the point grid
+ *   (393) is coarser than the 0-1000 grid, so a points round-trip double-rounds
+ *   (500→197→501). Pixels (1179 > 1000) are finer → identity. The vision model
+ *   is resolution-independent (0-1000 is a fraction of the image), so the
+ *   backend never converts coords with screen_width/height.
+ */
+import { resolveTextValue } from "./actions.js";
+import { requireOneBootedSimulator, screenshotPng, terminateApp, launchApp, installApp, isAppInstalled, bundleIdFromApp, } from "./simctl.js";
+// iOS UI interaction + a11y run through WebDriverAgent (XCUITest), not idb.
+import { ensureWda, closeWda, describeScreen, describeAll, uiTap, uiLongPress, uiSwipe, uiText, uiKey, HID_KEY_RETURN, } from "./xcuitest.js";
+import { isLocalPath } from "../upload.js";
+import { deNormalizePoint, deNormalizeDrag, pointToPixel } from "./coordinates.js";
+import { parseXcuiHierarchy, serializeNativeTree, boundsCenter } from "./native-a11y.js";
+// Let animations/transitions settle before the next observation so the
+// screenshot the LLM reasons over reflects the action's result.
+const POST_GESTURE_SETTLE_MS = 500;
+// Leading nav-bar labels that are NOT a back affordance — used to keep
+// navigate_back's back-button resolver from tapping a destructive/wrong
+// control (Cancel/Close discard work; Edit/Done/Add/Save/Menu are actions,
+// not navigation). A stock back button is labeled with the parent screen's
+// title, so it never collides with this set.
+const NON_BACK_LEADING_LABELS = new Set([
+    "cancel",
+    "close",
+    "done",
+    "edit",
+    "add",
+    "save",
+    "menu",
+]);
+async function settle(ms = POST_GESTURE_SETTLE_MS) {
+    await new Promise((r) => setTimeout(r, ms));
+}
+export class IOSDevice {
+    contextValues;
+    log;
+    bundleId;
+    appPath;
+    /** udid of the single booted simulator we drive. */
+    udid = "";
+    /** Set once the WebDriverAgent runner is up, so the startup note logs once. */
+    wdaStarted = false;
+    /** POINT size — what idb ui tap/swipe consume (de-normalization basis for TAPS). */
+    pointWidth = 0;
+    pointHeight = 0;
+    /**
+     * PIXEL size — the screenshot resolution and the RECORDED coord space.
+     * Recording in pixels (not points) keeps the loop's round-trip exact: the
+     * point grid (e.g. 393) is coarser than the 0-1000 normalized grid, so a
+     * points round-trip double-rounds and drifts; pixels (e.g. 1179 > 1000) are
+     * finer, so de-normalize-then-re-normalize is an identity.
+     */
+    pixelWidth = 0;
+    pixelHeight = 0;
+    /**
+     * shortId → bounds (POINTS — idb describe-all frames) from the last observe(),
+     * the local counterpart of BrowserDevice.lastTreeData. executeAction()
+     * resolves a backend `node_id` against this; the bounds-center is the POINT
+     * tap target (recorded in pixels via pointToPixel).
+     */
+    lastNodeMap = new Map();
+    constructor(opts) {
+        this.contextValues = opts.contextValues;
+        this.log = opts.log ?? (() => { });
+        this.bundleId = opts.bundleId ?? null;
+        this.appPath = opts.appPath;
+    }
+    async launchOrReset(target) {
+        this.udid = await requireOneBootedSimulator();
+        // First call: install the .app (if --app is a local path) and resolve the
+        // bundle id to terminate/relaunch on. `target` is the iteration's platform
+        // target (a bundle id) when no --app is supplied. Throws (rather than
+        // silently driving the foreground) if the bundle id can't be resolved.
+        if (!this.bundleId) {
+            this.bundleId = await this.resolveBundleId(target);
+        }
+        const bundleId = this.bundleId;
+        // Bring up the WebDriverAgent runner (install + simctl-launch the prebuilt
+        // xctrunner, open a session). Idempotent and reused across participants, so
+        // the ~30-60s first-launch cost is paid once per run.
+        if (!this.wdaStarted) {
+            this.log("Starting the iOS automation runner (WebDriverAgent); first launch can take ~30-60s...");
+        }
+        await ensureWda(this.udid);
+        this.wdaStarted = true;
+        // Prime screen geometry (points) before the first de-normalization.
+        await this.refreshScreen();
+        // Per-participant reset: terminate then relaunch from a clean state.
+        await terminateApp(this.udid, bundleId);
+        await launchApp(this.udid, bundleId);
+        await settle(1500); // cold start needs longer than a gesture settle
+    }
+    /**
+     * Resolve the bundle id to drive, returning a non-null id or throwing.
+     * Installs a local `.app` first and reads its CFBundleIdentifier from
+     * Info.plist (no list-diff needed — a .app carries its id). A non-.app local
+     * value is treated as an already-installed bundle id.
+     */
+    async resolveBundleId(target) {
+        const appSpec = this.appPath ?? (target && target.trim() ? target.trim() : null);
+        if (!appSpec) {
+            throw new Error("No app to drive: pass --app <path-to.app | installed.bundle.id>, or set the iteration's " +
+                "platform target to an installed bundle id.");
+        }
+        // `isLocalPath` returns false for http(s):// and throws on other schemes.
+        const local = isLocalPath(appSpec);
+        if (!local) {
+            throw new Error(`--app received a URL (${appSpec}). Installing a hosted .app on the simulator is not supported yet — ` +
+                `pass a local .app path or an already-installed bundle id.`);
+        }
+        if (appSpec.toLowerCase().endsWith(".app")) {
+            const id = await bundleIdFromApp(appSpec);
+            if (!id) {
+                throw new Error(`Could not read CFBundleIdentifier from "${appSpec}/Info.plist". ` +
+                    `Pass --app <bundle.id> explicitly if the .app layout is unusual.`);
+            }
+            this.log(`Installing ${appSpec} (${id})...`);
+            await installApp(this.udid, appSpec);
+            return id;
+        }
+        // Local non-.app value: treat as an installed bundle id.
+        if (await isAppInstalled(this.udid, appSpec)) {
+            return appSpec;
+        }
+        throw new Error(`App "${appSpec}" is not installed on the simulator and is not a local .app path. ` +
+            `Pass --app <path-to.app> to install it, or install it first.`);
+    }
+    async refreshScreen() {
+        const screen = await describeScreen(this.udid);
+        this.pointWidth = screen.pointWidth;
+        this.pointHeight = screen.pointHeight;
+        this.pixelWidth = screen.pixelWidth;
+        this.pixelHeight = screen.pixelHeight;
+        return screen;
+    }
+    async observe() {
+        // Refresh geometry each step (orientation can change), then capture the
+        // pixel screenshot and the a11y tree in parallel (independent reads). The
+        // dump is wrapped so a failure degrades to the vision path (empty tree).
+        await this.refreshScreen();
+        const [png, tree] = await Promise.all([
+            screenshotPng(),
+            this.dumpTree(),
+        ]);
+        this.lastNodeMap = tree.nodeMap;
+        return {
+            screenshot: png.toString("base64"),
+            // Element path when describe-all produced a tree; "" → backend vision.
+            accessibilityTree: tree.simplified,
+            url: "",
+            // PIXELS — match dimensions() and the pixel screenshot we send, so the
+            // loop's coordinate round-trip is exact (see dimensions()/toPixels()).
+            width: this.pixelWidth,
+            height: this.pixelHeight,
+            // Native has no scrollable document; the screen IS the page.
+            documentHeight: this.pixelHeight,
+            tabs: [],
+        };
+    }
+    /**
+     * Read + serialize WDA's /source a11y tree (bounds in POINTS). Any
+     * failure (retries exhausted on a trivial tree, parse error) degrades to an
+     * empty tree so the backend falls back to vision — a missing tree must never
+     * abort the observation.
+     */
+    async dumpTree() {
+        try {
+            const json = await describeAll(this.udid);
+            const nodes = parseXcuiHierarchy(json);
+            const tree = serializeNativeTree(nodes);
+            this.log(`a11y tree: ${tree.nodeMap.size} node(s)`);
+            return tree;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            this.log(`a11y describe-all failed, falling back to vision: ${msg}`);
+            return { simplified: "", nodeMap: new Map() };
+        }
+    }
+    async captureScreenshot() {
+        const png = await screenshotPng();
+        return png.toString("base64");
+    }
+    async captureScreenshotJpeg() {
+        // simctl screenshot only emits PNG. We return the PNG bytes; the upload/
+        // record path treats them as opaque image bytes (PDQ frame-matching works
+        // on PNG). The loop labels native uploads image/png.
+        return screenshotPng();
+    }
+    dimensions() {
+        // PIXELS — the space the loop re-normalizes the recorded coord against.
+        // Pixels (finer than the 0-1000 grid) make that round-trip exact; idb taps
+        // separately in points (see toPoints()).
+        return { width: this.pixelWidth, height: this.pixelHeight };
+    }
+    /** Normalized 0-1000 → POINT space (WDA taps/swipes take points). */
+    toPoints(c) {
+        return deNormalizePoint(c, this.pointWidth, this.pointHeight);
+    }
+    /** Normalized 0-1000 → PIXEL space (the recorded/reported coord). */
+    toPixels(c) {
+        return deNormalizePoint(c, this.pixelWidth, this.pixelHeight);
+    }
+    /**
+     * Resolve the POINT tap target + PIXEL record coord for a positional action.
+     * ELEMENT path (node_id): the bounds-center is the POINT tap; the recorded
+     * pixel coord is that center scaled POINTS→PIXELS so it round-trips against
+     * dimensions() (pixels). VISION path: de-normalize the 0-1000 coord into both
+     * spaces. Returns {stale:true} for a node_id with no bounds (tree moved); the
+     * caller fails the action so the loop forwards DOM_ELEMENT_NOT_FOUND.
+     */
+    resolveTarget(action) {
+        if (action.node_id) {
+            const bounds = this.lastNodeMap.get(action.node_id);
+            if (!bounds)
+                return { pt: null, px: null, stale: true };
+            const pt = boundsCenter(bounds); // POINTS — WDA taps directly
+            const px = pointToPixel(pt, this.pointWidth, this.pointHeight, this.pixelWidth, this.pixelHeight);
+            return { pt, px, stale: false };
+        }
+        const pt = action.coordinates ? this.toPoints(action.coordinates) : null;
+        const px = action.coordinates ? this.toPixels(action.coordinates) : null;
+        return { pt, px, stale: false };
+    }
+    async executeAction(action) {
+        try {
+            // pt drives the WDA TAP (points); px is what we RECORD (pixels). ELEMENT
+            // path: pt = bounds-center, px = that center scaled to pixels. VISION
+            // path: both derive from the same normalized coord. Either way the tap
+            // lands right and the recorded px round-trips against dimensions().
+            const resolved = this.resolveTarget(action);
+            if (resolved.stale)
+                return this.failStaleNode(action);
+            const { pt, px } = resolved;
+            switch (action.type) {
+                case "tap":
+                case "double_tap": {
+                    if (!pt)
+                        return this.failNoCoords(action);
+                    const count = action.type === "double_tap" ? 2 : action.count ?? 1;
+                    for (let i = 0; i < count; i++)
+                        await uiTap(this.udid, pt.x, pt.y);
+                    break;
+                }
+                case "long_press": {
+                    if (!pt)
+                        return this.failNoCoords(action);
+                    await uiLongPress(this.udid, pt.x, pt.y, action.duration_ms ?? 600);
+                    break;
+                }
+                case "text_input": {
+                    await this.typeText(action, pt);
+                    break;
+                }
+                case "scroll": {
+                    await this.scroll(action);
+                    break;
+                }
+                case "swipe":
+                case "pull_to_refresh": {
+                    await this.swipe(action.direction ?? (action.type === "pull_to_refresh" ? "down" : "up"));
+                    break;
+                }
+                case "navigate_back": {
+                    // iOS has no hardware back; the system "back" is a left-edge swipe.
+                    await this.navigateBack();
+                    break;
+                }
+                case "open_system_panel": {
+                    // Element-less, like navigate_back: best-effort top-edge pull-down.
+                    await this.openSystemPanel(action.panel === "quick_settings" ? "quick_settings" : "notifications");
+                    break;
+                }
+                case "drag": {
+                    // A drag GRABS an element and RELEASES it elsewhere ("click the
+                    // element, move, let go") — distinct from a swipe (element-less
+                    // directional). Press the resolved element center (pt — the same
+                    // element path a tap uses, in POINTS), move to the drop point,
+                    // release. Record the grab point→PIXELS so it round-trips against
+                    // dimensions() (pixels).
+                    const recorded = await this.drag(action, pt);
+                    if (!recorded)
+                        return this.failNoCoords(action);
+                    await settle();
+                    return {
+                        success: true,
+                        elementName: action.element_name,
+                        coordinates: recorded,
+                        openedNewTab: false,
+                    };
+                }
+                case "wait": {
+                    await settle(action.duration_ms ?? 1000);
+                    break;
+                }
+                case "think": {
+                    // Reasoning-only: no device interaction.
+                    return { success: true, elementName: action.element_name, coordinates: null, openedNewTab: false };
+                }
+                case "pinch_zoom":
+                case "rotate_device":
+                case "keyboard_shortcut":
+                case "switch_tab":
+                case "close_tab": {
+                    // Capabilities the single-app iOS driver genuinely can't perform:
+                    // true multi-touch pinch, sensor rotation (idb has no clean rotate),
+                    // browser tabs, desktop keyboard shortcuts. Fail LOUDLY (not silently)
+                    // so the loop forwards it and the agent can adapt.
+                    return this.failUnsupported(action);
+                }
+                default: {
+                    this.log(`Unknown native action: ${action.type}`);
+                    return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+                }
+            }
+            await settle();
+            return {
+                success: true,
+                elementName: action.element_name,
+                // Report PIXEL coords so the loop re-normalizes them against the pixel
+                // dimensions() (points would drift since 393 < 1000). The tap itself
+                // used points — either the de-normalized 0-1000 (vision) or the
+                // bounds-center (element); px is the matching pixel coord for each.
+                coordinates: px,
+                openedNewTab: false,
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            this.log(`Action ${action.type} failed: ${msg}`);
+            return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+        }
+    }
+    failNoCoords(action) {
+        // The backend couldn't vision-locate a target (coordinates=null). Skip the
+        // action (don't crash, don't silently succeed) and surface it like the
+        // browser path's unresolved-element case — success:false makes the loop
+        // push a DOM_ELEMENT_NOT_FOUND forward so the LLM learns the target missed.
+        const target = action.element_description || action.element_name || "(no description)";
+        this.log(`Skipping native action with no resolved target: ${action.type} ${target}`);
+        return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+    }
+    failStaleNode(action) {
+        // The backend resolved a node_id that's no longer in the latest tree (the
+        // screen changed between observe() and act). Fail like the browser's
+        // unresolved-element case — success:false forwards DOM_ELEMENT_NOT_FOUND so
+        // the loop re-observes and the agent retries against a fresh tree.
+        const target = action.element_description || action.element_name || action.node_id || "(unknown)";
+        this.log(`Stale node_id "${action.node_id}" not in current a11y tree: ${action.type} ${target}`);
+        return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+    }
+    async typeText(action, pt) {
+        // Focus the field first if the model gave a target.
+        if (pt) {
+            await uiTap(this.udid, pt.x, pt.y);
+            await settle(250);
+        }
+        const text = resolveTextValue(action, this.contextValues);
+        // WDA text input appends to the focused field; for click_type (replace) there
+        // is no idb "clear", so we rely on the field being empty after focus. The
+        // vision agent typically taps an empty field, so this matches Android's
+        // common path; a true select-all clear isn't exposed by idb.
+        if (text)
+            await uiText(this.udid, text);
+        if (action.submit) {
+            await settle(150);
+            await uiKey(this.udid, HID_KEY_RETURN);
+        }
+    }
+    async scroll(action) {
+        const w = this.pointWidth;
+        const h = this.pointHeight;
+        const cx = Math.round(w / 2);
+        const amountMap = {
+            small: 0.25, medium: 0.45, large: 0.7, extra_large: 0.9,
+        };
+        const frac = amountMap[action.amount ?? "medium"] ?? 0.45;
+        const dist = Math.round(h * frac);
+        const mid = Math.round(h / 2);
+        switch (action.direction) {
+            case "up":
+                // Reveal content above: swipe finger downward.
+                await uiSwipe(this.udid, cx, mid - dist / 2, cx, mid + dist / 2);
+                break;
+            case "to_top":
+                await uiSwipe(this.udid, cx, Math.round(h * 0.2), cx, Math.round(h * 0.9), 400);
+                break;
+            case "to_bottom":
+                await uiSwipe(this.udid, cx, Math.round(h * 0.9), cx, Math.round(h * 0.2), 400);
+                break;
+            case "down":
+            case "to_element":
+            default:
+                // Reveal content below: swipe finger upward.
+                await uiSwipe(this.udid, cx, mid + dist / 2, cx, mid - dist / 2);
+                break;
+        }
+    }
+    async swipe(direction) {
+        const w = this.pointWidth;
+        const h = this.pointHeight;
+        const cx = Math.round(w / 2);
+        const cy = Math.round(h / 2);
+        const d = Math.round(h * 0.4);
+        const dx = Math.round(w * 0.4);
+        switch (direction) {
+            case "up":
+                await uiSwipe(this.udid, cx, cy + d / 2, cx, cy - d / 2);
+                break;
+            case "down":
+                await uiSwipe(this.udid, cx, cy - d / 2, cx, cy + d / 2);
+                break;
+            case "left":
+                await uiSwipe(this.udid, cx + dx / 2, cy, cx - dx / 2, cy);
+                break;
+            case "right":
+                await uiSwipe(this.udid, cx - dx / 2, cy, cx + dx / 2, cy);
+                break;
+            default:
+                await uiSwipe(this.udid, cx, cy + d / 2, cx, cy - d / 2);
+                break;
+        }
+    }
+    /**
+     * Perform a drag: press the GRABBED element, move to the drop point, release.
+     * A drag is "click an element and let it go", so the press lands element-
+     * center (the resolved `grab` in POINTS — node_id bounds center, or the
+     * vision coordinate when the tree is blind), NOT the backend's vision-
+     * estimated start. The release point is the drag END (drag.endX/endY). A
+     * ~0.8s idb swipe reads as a drag, not a flick. Returns the grab point scaled
+     * to PIXELS (pointToPixel) to record so it round-trips against dimensions()
+     * (pixels), or null if there's no end to drag toward.
+     *
+     * idb LIMITATION: `idb ui swipe` only exposes --duration/--delta — it has no
+     * press-and-HOLD-then-move primitive (unlike Android's `input draganddrop`).
+     * So this drives the immediate-drag surfaces (sliders, drag-to-dismiss, drag
+     * handles that pick up on touch-move) but does NOT trigger a long-press
+     * pickup (home-screen jiggle mode, in-app reorder that needs a hold first) —
+     * verified on-device: a long uiSwipe leaves home-screen icons unmoved. The
+     * grab/release SEMANTICS are still correct; the gap is purely the missing
+     * hold, which idb can't perform in one continuous gesture.
+     */
+    async drag(action, grab) {
+        if (!action.drag)
+            return null;
+        const { start, end } = deNormalizeDrag(action.drag, this.pointWidth, this.pointHeight); // POINTS
+        // Grab the resolved element center; fall back to the backend's own start
+        // only when nothing resolved (no node_id and no vision coordinate).
+        const press = grab ?? start;
+        await uiSwipe(this.udid, press.x, press.y, end.x, end.y, 800);
+        return pointToPixel(press, this.pointWidth, this.pointHeight, this.pixelWidth, this.pixelHeight);
+    }
+    /**
+     * iOS has no hardware back. The system interactive-pop (left-edge swipe) is
+     * NOT reliably triggerable through idb's synthetic touch — verified on the
+     * simulator: no edge-swipe variant (start x, travel, duration, delta) pops
+     * the view. So we resolve and TAP the nav-bar back button instead: iOS HIG
+     * places "back" as the LEADING (leftmost) button in the top nav bar of any
+     * pushed view, so the leftmost button in the nav-bar band is it — verified to
+     * pop a Settings sub-screen back to root. The left-edge swipe remains a
+     * best-effort fallback for real devices (where idb sends real HID events that
+     * do drive the system gesture) when no back button is visible.
+     */
+    async navigateBack() {
+        const nodes = parseXcuiHierarchy(await describeAll(this.udid));
+        const back = this.findBackButton(nodes);
+        if (back) {
+            const c = boundsCenter(back.bounds); // POINTS — WDA taps directly
+            await uiTap(this.udid, c.x, c.y);
+            return;
+        }
+        // No nav-bar back button (root screen, or a custom chrome): fall back to the
+        // system edge-swipe — works on real devices, a no-op on the simulator.
+        this.log("navigate_back: no nav-bar back button found; trying left-edge swipe");
+        const midY = Math.round(this.pointHeight / 2);
+        await uiSwipe(this.udid, 1, midY, Math.round(this.pointWidth * 0.5), midY, 300);
+    }
+    /**
+     * Best-effort open of an iOS system panel by swiping down from the top edge.
+     * iOS has no `cmd statusbar` equivalent, so on a Face-ID layout:
+     *   - notifications → Notification Center: swipe down from the top-CENTER.
+     *   - quick_settings → Control Center: swipe down from the top-RIGHT corner.
+     * Coordinates are POINTS (idb consumes points; see toPoints()/the swipe()
+     * helper). This is FLAKY on the simulator — idb's synthetic touch frequently
+     * doesn't trigger the system edge gesture (the same limitation navigateBack's
+     * edge-swipe hits). We compare a before/after screenshot and log LOUDLY when
+     * the screen didn't change, rather than silently reporting success, so a
+     * no-op is visible in the run. The executeAction caller still returns
+     * success:true (the gesture was attempted); the loud log is the signal.
+     */
+    async openSystemPanel(panel) {
+        const before = await screenshotPng();
+        const w = this.pointWidth;
+        const h = this.pointHeight;
+        // Start ON the top edge and travel a third of the screen down. Control
+        // Center lives under the top-right (battery/status) corner on Face-ID
+        // devices; Notification Center under the top-center notch area.
+        const startX = panel === "quick_settings" ? Math.round(w * 0.92) : Math.round(w * 0.5);
+        const startY = 1;
+        const endY = Math.round(h * 0.35);
+        await uiSwipe(this.udid, startX, startY, startX, endY, 350);
+        await settle();
+        // Loudly surface a no-op: the simulator's synthetic touch often can't drive
+        // the system edge gesture. An identical screenshot means the panel didn't open.
+        const after = await screenshotPng();
+        if (before.equals(after)) {
+            this.log(`open_system_panel (${panel}): top-edge swipe produced no visible change — ` +
+                `the simulator's synthetic touch likely didn't trigger the system gesture (flaky on the simulator).`);
+        }
+    }
+    /**
+     * The nav-bar back button: the leading (leftmost) actionable button in the
+     * top nav-bar band. iOS HIG guarantees "back" is the leading nav item in a
+     * pushed view, so the leftmost button high on the screen is it. Returns null
+     * on root screens (no leading back item) so the caller can fall back.
+     *
+     * The geometry alone (leftmost-top) would mis-fire on a modal whose LEADING
+     * item is Cancel/Close, or a root with a leading Edit/menu — and tapping
+     * Cancel/Close can DISCARD work. A stock back button is labeled with the
+     * PARENT screen's title (e.g. "Settings"), not "Back", so there's no reliable
+     * positive label signal; instead we exclude the known non-back leading
+     * labels. If every leading button is one of those, we return null and let the
+     * caller fall back rather than tap a destructive control.
+     *
+     * Known limitation: a glyph-only leading button with NO accessible label
+     * (e.g. a hamburger/avatar/logo) isn't in the deny-list, so on a screen whose
+     * leading control is an unlabeled non-back icon this can tap the wrong control
+     * (silently — it returns success). Acceptable for the common case (stock nav
+     * bars have a labeled back button), but it's why pushed views, not root/menu
+     * screens, are where navigate_back is reliable.
+     */
+    findBackButton(nodes) {
+        const navBandBottom = this.pointHeight * 0.15;
+        const leftZone = this.pointWidth * 0.3;
+        const candidates = nodes.filter((n) => n.role === "button" &&
+            n.clickable &&
+            n.bounds.width > 0 &&
+            n.bounds.height > 0 &&
+            n.bounds.y < navBandBottom &&
+            n.bounds.x < leftZone &&
+            !NON_BACK_LEADING_LABELS.has(n.label.trim().toLowerCase()));
+        if (candidates.length === 0)
+            return null;
+        // Leftmost wins; tie-break by topmost.
+        candidates.sort((a, b) => a.bounds.x - b.bounds.x || a.bounds.y - b.bounds.y);
+        return candidates[0];
+    }
+    failUnsupported(action) {
+        // A capability the single-app iOS driver genuinely can't perform. Fail with
+        // a clear, diagnosable reason — NOT a silent false — so it's visible WHY and
+        // the agent can adapt.
+        const hint = {
+            pinch_zoom: "no multi-touch on the native driver; the agent should zoom via double_tap",
+            rotate_device: "rotation is not wired on the native driver; leave orientation as-is",
+            keyboard_shortcut: "no hardware keyboard on the native driver; use on-screen taps/text_input",
+            switch_tab: "tabs are a browser concept; the native app has a single window",
+            close_tab: "tabs are a browser concept; the native app has a single window",
+        };
+        this.log(`${action.type} not supported by the iOS native driver — ${hint[action.type] ?? "no native equivalent"}`);
+        return { success: false, elementName: action.element_name, coordinates: null, openedNewTab: false };
+    }
+    currentUrl() {
+        // Native has no URL; recording stores "" (current_location comes from the
+        // backend's reasoning output, not the device).
+        return "";
+    }
+    async close() {
+        // Tear down the WebDriverAgent session (the runner is left installed on the
+        // shared simulator for the next run). The app resets via launchOrReset; no
+        // IME state to restore on iOS.
+        await closeWda(this.udid);
+    }
+}

package/dist/lib/local-sim/loop.d.ts

@@ -1,10 +1,12 @@
 /**
  * Local simulation loop orchestrator.
  *
- * Runs the observe → reason (remote) → act (local) loop for each
- * participant against a local Playwright browser.
+ * Runs the observe → reason (remote) → act (local) loop for each participant
+ * against a SimulationDevice (a Playwright browser today; a native Android
+ * emulator next). The loop is device-agnostic — see device.ts.
  */
 import type { ApiClient } from "../api-client.js";
+import type { LocalStepAction } from "./types.js";
 export interface DebugStep {
     step: number;
     assignmentName: string;
@@ -35,6 +37,14 @@ export interface DebugStep {
     assignmentCompleted: boolean;
     effortSeconds: number;
 }
+/**
+ * Convert a raw action (from either resolved_actions or output.action.actions)
+ * into the flat LocalStepAction shape used by the executor. Exported for unit
+ * tests of the native drag coordinate-shape split (the nested action's
+ * `coordinates` is a {x,y} tap point for most actions but a
+ * {startX,...,endY} path for a drag).
+ */
+export declare function flattenAction(raw: Record<string, unknown>, nodeId?: string | null, nodeDescription?: string | null): LocalStepAction;
 export interface LocalSimRunOptions {
     workspaceId: string;
     studyId: string;
@@ -52,6 +62,8 @@ export interface LocalSimRunOptions {
     json?: boolean;
     debug?: boolean;
     parallel?: number;
+    platform?: string;
+    appPath?: string;
 }
 /**
  * Run local simulations — parallel when multiple participants, sequential by default.