@ishlabs/cli 0.24.1 → 0.26.0

package/dist/lib/local-sim/device.js

@@ -0,0 +1,152 @@
+/**
+ * SimulationDevice — the target a local simulation drives.
+ *
+ * The observe → reason (remote) → act (local) loop in `loop.ts` used to be
+ * hardwired to a Playwright `Page`. This interface abstracts exactly what the
+ * loop needs from a target so a native Android device (driven by `adb`) can
+ * slot in next to the browser. `BrowserDevice` (below) wraps the existing
+ * Playwright path in `browser.ts`/`actions.ts`/`tabs.ts`; `AndroidDevice`
+ * (added later) implements the same surface via `adb`.
+ *
+ * Multi-tab handling is browser-specific and stays hidden behind the
+ * interface — the loop never touches a `Page` or `TabManager` directly.
+ */
+import { launchBrowser, createTab, captureObservation, takeScreenshot, takeScreenshotJpeg, takeFullPageJpeg, navigateWithRetry, closeBrowser, } from "./browser.js";
+import { executeAction } from "./actions.js";
+import { TabManager } from "./tabs.js";
+import { debugObservation } from "./debug.js";
+/**
+ * Browser implementation backed by Playwright. Delegates to the existing
+ * `browser.ts`/`actions.ts`/`tabs.ts` helpers — no logic is rewritten here.
+ *
+ * Owns a `BrowserSession` plus a `TabManager`; the active page can swap when a
+ * popup auto-focuses or the LLM issues switch_tab/close_tab, so every method
+ * re-reads `tabs.activePage()` before acting (matching the previous loop).
+ */
+export class BrowserDevice {
+    session;
+    tabs;
+    opts;
+    contextValues;
+    /** When false this device shares a browser process and only closes its tab. */
+    ownsBrowser;
+    /** CDP node map from the last observe(), needed to resolve actions. */
+    lastTreeData = null;
+    constructor(session, opts, contextValues, ownsBrowser) {
+        this.session = session;
+        this.opts = opts;
+        this.contextValues = contextValues;
+        this.ownsBrowser = ownsBrowser;
+        this.tabs = new TabManager(session.context, session.page);
+    }
+    async launchOrReset(target) {
+        await navigateWithRetry(this.tabs.activePage(), target);
+    }
+    async observe() {
+        const page = this.tabs.activePage();
+        const obs = await captureObservation(page);
+        this.lastTreeData = obs.treeData;
+        debugObservation(obs);
+        const tabsSnapshot = await this.tabs.list();
+        return {
+            screenshot: obs.screenshot,
+            accessibilityTree: obs.treeData.simplified,
+            url: obs.url,
+            width: obs.viewportWidth,
+            height: obs.viewportHeight,
+            documentHeight: obs.documentHeight,
+            tabs: tabsSnapshot,
+        };
+    }
+    async captureScreenshot() {
+        return takeScreenshot(this.tabs.activePage());
+    }
+    async captureScreenshotJpeg() {
+        return takeScreenshotJpeg(this.tabs.activePage());
+    }
+    async captureFullPageJpeg(opts) {
+        const page = this.tabs.activePage();
+        const viewportWidth = page.viewportSize()?.width ?? this.opts.viewport.width;
+        const fullPage = await takeFullPageJpeg(page, {
+            documentHeight: opts.documentHeight,
+            cap: opts.cap,
+            viewportWidth,
+        });
+        return fullPage.base64;
+    }
+    dimensions() {
+        const page = this.tabs.activePage();
+        return page.viewportSize() ?? this.opts.viewport;
+    }
+    async executeAction(action) {
+        // Pick up popup auto-switch / explicit tab switch from prior actions.
+        let page = this.tabs.activePage();
+        const treeData = this.lastTreeData ?? { simplified: "", nodeMap: new Map() };
+        const tabsBefore = (await this.tabs.list()).length;
+        const result = await executeAction(page, action, treeData, this.contextValues, this.tabs);
+        // The action may have flipped the active tab — re-read.
+        page = this.tabs.activePage();
+        const tabsAfter = (await this.tabs.list()).length;
+        const openedNewTab = action.type === "tap" && tabsAfter > tabsBefore;
+        return {
+            success: result.success,
+            elementName: result.elementName,
+            coordinates: result.coordinates,
+            openedNewTab,
+        };
+    }
+    currentUrl() {
+        return this.tabs.activePage().url();
+    }
+    async close() {
+        if (this.ownsBrowser) {
+            await closeBrowser(this.session);
+        }
+        else {
+            // Shared mode: close just the tab, not the context or browser.
+            try {
+                await this.session.page.close();
+            }
+            catch {
+                // already closed
+            }
+        }
+    }
+}
+/**
+ * Build the device for a platform. `web`/`browser`/`""` → Playwright
+ * `BrowserDevice`; `android` → `AndroidDevice` (adb); `ios` → `IOSDevice`
+ * (simctl + WebDriverAgent). The native cases are dynamically imported so the browser path
+ * never pulls in the adb/simctl modules.
+ */
+export async function createDevice(platform, opts) {
+    switch (platform) {
+        case "web":
+        case "browser":
+        case "": {
+            const ownsBrowser = !opts.sharedBrowser;
+            const session = opts.sharedBrowser
+                ? await createTab(opts.sharedBrowser, opts.browserOpts)
+                : await launchBrowser(opts.browserOpts);
+            return new BrowserDevice(session, opts.browserOpts, opts.contextValues, ownsBrowser);
+        }
+        case "android": {
+            const { AndroidDevice } = await import("./android.js");
+            return new AndroidDevice({
+                appPath: opts.appPath,
+                contextValues: opts.contextValues,
+                log: opts.log,
+            });
+        }
+        case "ios": {
+            const { IOSDevice } = await import("./ios.js");
+            return new IOSDevice({
+                appPath: opts.appPath,
+                contextValues: opts.contextValues,
+                log: opts.log,
+            });
+        }
+        default:
+            throw new Error(`Unsupported platform for local simulation: "${platform}"`);
+    }
+}

package/dist/lib/local-sim/ios.d.ts

@@ -0,0 +1,185 @@
+/**
+ * IOSDevice — drives a local iOS simulator via `xcrun simctl` (lifecycle +
+ * screenshot) and WebDriverAgent/XCUITest (UI + a11y; see xcuitest.ts),
+ * implementing the SimulationDevice surface the loop expects. Mirrors
+ * AndroidDevice; the one substantive difference is the coordinate space.
+ *
+ * Two resolution paths, mirroring the browser:
+ *  - ELEMENT (preferred): observe() reads WDA's `/source` a11y tree, serializes
+ *    it to the `[id] role "label"` string the backend DOMLocator reasons over,
+ *    and keeps a local `shortId → bounds` map (bounds in POINTS). The backend
+ *    returns a `node_id`; executeAction() looks the bounds up and taps the
+ *    element's CENTER.
+ *  - VISION (fallback): when the tree is empty/sparse, observe() returns an
+ *    empty tree so the backend takes its vision branch and returns NORMALIZED
+ *    0-1000 coordinates. Also taken per-action whenever node_id is absent.
+ *
+ * COORDINATE SPACE — two spaces, the key difference from Android (where
+ * screencap and tap share one pixel space):
+ *   `simctl io booted screenshot` is in PIXELS (e.g. 1179x2556 @3x), but
+ *   WDA taps/swipes AND the `/source` a11y frames are POINTS (393x852).
+ *   The invariant in BOTH paths: TAP in points, RECORD in pixels, because the
+ *   loop re-normalizes the recorded coord against dimensions() (PIXELS).
+ *     - VISION:  tap pt = round(n/1000 * pointSize); record px = round(n/1000 * pixelSize).
+ *     - ELEMENT: tap = bounds-center (already POINTS); record = that center
+ *       scaled POINTS→PIXELS via pointToPixel() (the @Nx scale).
+ *   dimensions() returns the PIXEL size, so the loop re-normalizes the recorded
+ *   px back to a stable 0-1000. Recording in points would drift: the point grid
+ *   (393) is coarser than the 0-1000 grid, so a points round-trip double-rounds
+ *   (500→197→501). Pixels (1179 > 1000) are finer → identity. The vision model
+ *   is resolution-independent (0-1000 is a fraction of the image), so the
+ *   backend never converts coords with screen_width/height.
+ */
+import type { LocalStepAction, ContextValue } from "./types.js";
+import type { SimulationDevice, DeviceObservation, DeviceActionResult } from "./device.js";
+export interface IosDeviceOptions {
+    /** Bundle id to terminate/relaunch between participants. Derived from --app when a .app is given. */
+    bundleId?: string;
+    /** Local .app path to install before the run, or a bundle id to launch. */
+    appPath?: string;
+    contextValues: ContextValue[];
+    log?: (msg: string) => void;
+}
+export declare class IOSDevice implements SimulationDevice {
+    private readonly contextValues;
+    private readonly log;
+    private bundleId;
+    private readonly appPath;
+    /** udid of the single booted simulator we drive. */
+    private udid;
+    /** Set once the WebDriverAgent runner is up, so the startup note logs once. */
+    private wdaStarted;
+    /** POINT size — what idb ui tap/swipe consume (de-normalization basis for TAPS). */
+    private pointWidth;
+    private pointHeight;
+    /**
+     * PIXEL size — the screenshot resolution and the RECORDED coord space.
+     * Recording in pixels (not points) keeps the loop's round-trip exact: the
+     * point grid (e.g. 393) is coarser than the 0-1000 normalized grid, so a
+     * points round-trip double-rounds and drifts; pixels (e.g. 1179 > 1000) are
+     * finer, so de-normalize-then-re-normalize is an identity.
+     */
+    private pixelWidth;
+    private pixelHeight;
+    /**
+     * shortId → bounds (POINTS — idb describe-all frames) from the last observe(),
+     * the local counterpart of BrowserDevice.lastTreeData. executeAction()
+     * resolves a backend `node_id` against this; the bounds-center is the POINT
+     * tap target (recorded in pixels via pointToPixel).
+     */
+    private lastNodeMap;
+    constructor(opts: IosDeviceOptions);
+    launchOrReset(target: string): Promise<void>;
+    /**
+     * Resolve the bundle id to drive, returning a non-null id or throwing.
+     * Installs a local `.app` first and reads its CFBundleIdentifier from
+     * Info.plist (no list-diff needed — a .app carries its id). A non-.app local
+     * value is treated as an already-installed bundle id.
+     */
+    private resolveBundleId;
+    private refreshScreen;
+    observe(): Promise<DeviceObservation>;
+    /**
+     * Read + serialize WDA's /source a11y tree (bounds in POINTS). Any
+     * failure (retries exhausted on a trivial tree, parse error) degrades to an
+     * empty tree so the backend falls back to vision — a missing tree must never
+     * abort the observation.
+     */
+    private dumpTree;
+    captureScreenshot(): Promise<string>;
+    captureScreenshotJpeg(): Promise<Buffer>;
+    dimensions(): {
+        width: number;
+        height: number;
+    };
+    /** Normalized 0-1000 → POINT space (WDA taps/swipes take points). */
+    private toPoints;
+    /** Normalized 0-1000 → PIXEL space (the recorded/reported coord). */
+    private toPixels;
+    /**
+     * Resolve the POINT tap target + PIXEL record coord for a positional action.
+     * ELEMENT path (node_id): the bounds-center is the POINT tap; the recorded
+     * pixel coord is that center scaled POINTS→PIXELS so it round-trips against
+     * dimensions() (pixels). VISION path: de-normalize the 0-1000 coord into both
+     * spaces. Returns {stale:true} for a node_id with no bounds (tree moved); the
+     * caller fails the action so the loop forwards DOM_ELEMENT_NOT_FOUND.
+     */
+    private resolveTarget;
+    executeAction(action: LocalStepAction): Promise<DeviceActionResult>;
+    private failNoCoords;
+    private failStaleNode;
+    private typeText;
+    private scroll;
+    private swipe;
+    /**
+     * Perform a drag: press the GRABBED element, move to the drop point, release.
+     * A drag is "click an element and let it go", so the press lands element-
+     * center (the resolved `grab` in POINTS — node_id bounds center, or the
+     * vision coordinate when the tree is blind), NOT the backend's vision-
+     * estimated start. The release point is the drag END (drag.endX/endY). A
+     * ~0.8s idb swipe reads as a drag, not a flick. Returns the grab point scaled
+     * to PIXELS (pointToPixel) to record so it round-trips against dimensions()
+     * (pixels), or null if there's no end to drag toward.
+     *
+     * idb LIMITATION: `idb ui swipe` only exposes --duration/--delta — it has no
+     * press-and-HOLD-then-move primitive (unlike Android's `input draganddrop`).
+     * So this drives the immediate-drag surfaces (sliders, drag-to-dismiss, drag
+     * handles that pick up on touch-move) but does NOT trigger a long-press
+     * pickup (home-screen jiggle mode, in-app reorder that needs a hold first) —
+     * verified on-device: a long uiSwipe leaves home-screen icons unmoved. The
+     * grab/release SEMANTICS are still correct; the gap is purely the missing
+     * hold, which idb can't perform in one continuous gesture.
+     */
+    private drag;
+    /**
+     * iOS has no hardware back. The system interactive-pop (left-edge swipe) is
+     * NOT reliably triggerable through idb's synthetic touch — verified on the
+     * simulator: no edge-swipe variant (start x, travel, duration, delta) pops
+     * the view. So we resolve and TAP the nav-bar back button instead: iOS HIG
+     * places "back" as the LEADING (leftmost) button in the top nav bar of any
+     * pushed view, so the leftmost button in the nav-bar band is it — verified to
+     * pop a Settings sub-screen back to root. The left-edge swipe remains a
+     * best-effort fallback for real devices (where idb sends real HID events that
+     * do drive the system gesture) when no back button is visible.
+     */
+    private navigateBack;
+    /**
+     * Best-effort open of an iOS system panel by swiping down from the top edge.
+     * iOS has no `cmd statusbar` equivalent, so on a Face-ID layout:
+     *   - notifications → Notification Center: swipe down from the top-CENTER.
+     *   - quick_settings → Control Center: swipe down from the top-RIGHT corner.
+     * Coordinates are POINTS (idb consumes points; see toPoints()/the swipe()
+     * helper). This is FLAKY on the simulator — idb's synthetic touch frequently
+     * doesn't trigger the system edge gesture (the same limitation navigateBack's
+     * edge-swipe hits). We compare a before/after screenshot and log LOUDLY when
+     * the screen didn't change, rather than silently reporting success, so a
+     * no-op is visible in the run. The executeAction caller still returns
+     * success:true (the gesture was attempted); the loud log is the signal.
+     */
+    private openSystemPanel;
+    /**
+     * The nav-bar back button: the leading (leftmost) actionable button in the
+     * top nav-bar band. iOS HIG guarantees "back" is the leading nav item in a
+     * pushed view, so the leftmost button high on the screen is it. Returns null
+     * on root screens (no leading back item) so the caller can fall back.
+     *
+     * The geometry alone (leftmost-top) would mis-fire on a modal whose LEADING
+     * item is Cancel/Close, or a root with a leading Edit/menu — and tapping
+     * Cancel/Close can DISCARD work. A stock back button is labeled with the
+     * PARENT screen's title (e.g. "Settings"), not "Back", so there's no reliable
+     * positive label signal; instead we exclude the known non-back leading
+     * labels. If every leading button is one of those, we return null and let the
+     * caller fall back rather than tap a destructive control.
+     *
+     * Known limitation: a glyph-only leading button with NO accessible label
+     * (e.g. a hamburger/avatar/logo) isn't in the deny-list, so on a screen whose
+     * leading control is an unlabeled non-back icon this can tap the wrong control
+     * (silently — it returns success). Acceptable for the common case (stock nav
+     * bars have a labeled back button), but it's why pushed views, not root/menu
+     * screens, are where navigate_back is reliable.
+     */
+    private findBackButton;
+    private failUnsupported;
+    currentUrl(): string;
+    close(): Promise<void>;
+}