@ishlabs/cli 0.25.0 → 0.26.1

package/dist/lib/local-sim/ios.js

@@ -1,14 +1,15 @@
 /**
- * IOSDevice — drives a local iOS simulator via `xcrun simctl` + `idb`,
+ * IOSDevice — drives a local iOS simulator via `xcrun simctl` (lifecycle +
+ * screenshot) and WebDriverAgent/XCUITest (UI + a11y; see xcuitest.ts),
  * implementing the SimulationDevice surface the loop expects. Mirrors
  * AndroidDevice; the one substantive difference is the coordinate space.
  *
  * Two resolution paths, mirroring the browser:
- *  - ELEMENT (preferred): observe() reads the `idb ui describe-all` a11y tree,
- *    serializes it to the `[id] role "label"` string the backend DOMLocator
- *    reasons over, and keeps a local `shortId → bounds` map (bounds in POINTS).
- *    The backend returns a `node_id`; executeAction() looks the bounds up and
- *    taps the element's CENTER.
+ *  - ELEMENT (preferred): observe() reads WDA's `/source` a11y tree, serializes
+ *    it to the `[id] role "label"` string the backend DOMLocator reasons over,
+ *    and keeps a local `shortId → bounds` map (bounds in POINTS). The backend
+ *    returns a `node_id`; executeAction() looks the bounds up and taps the
+ *    element's CENTER.
  *  - VISION (fallback): when the tree is empty/sparse, observe() returns an
  *    empty tree so the backend takes its vision branch and returns NORMALIZED
  *    0-1000 coordinates. Also taken per-action whenever node_id is absent.
@@ -16,7 +17,7 @@
  * COORDINATE SPACE — two spaces, the key difference from Android (where
  * screencap and tap share one pixel space):
  *   `simctl io booted screenshot` is in PIXELS (e.g. 1179x2556 @3x), but
- *   `idb ui tap/swipe` AND the `describe-all` a11y frames are POINTS (393x852).
+ *   WDA taps/swipes AND the `/source` a11y frames are POINTS (393x852).
  *   The invariant in BOTH paths: TAP in points, RECORD in pixels, because the
  *   loop re-normalizes the recorded coord against dimensions() (PIXELS).
  *     - VISION:  tap pt = round(n/1000 * pointSize); record px = round(n/1000 * pixelSize).
@@ -30,10 +31,12 @@
  *   backend never converts coords with screen_width/height.
  */
 import { resolveTextValue } from "./actions.js";
-import { requireOneBootedSimulator, describeScreen, describeAll, screenshotPng, uiTap, uiLongPress, uiSwipe, uiText, uiKey, HID_KEY_RETURN, terminateApp, launchApp, installApp, isAppInstalled, bundleIdFromApp, } from "./simctl.js";
+import { requireOneBootedSimulator, screenshotPng, terminateApp, launchApp, installApp, isAppInstalled, bundleIdFromApp, appBuildFromSimulator, } from "./simctl.js";
+// iOS UI interaction + a11y run through WebDriverAgent (XCUITest), not idb.
+import { ensureWda, closeWda, describeScreen, describeAll, uiTap, uiLongPress, uiSwipe, uiText, uiKey, HID_KEY_RETURN, } from "./xcuitest.js";
 import { isLocalPath } from "../upload.js";
 import { deNormalizePoint, deNormalizeDrag, pointToPixel } from "./coordinates.js";
-import { parseIdbDescribeAll, serializeNativeTree, boundsCenter } from "./native-a11y.js";
+import { parseXcuiHierarchy, serializeNativeTree, boundsCenter } from "./native-a11y.js";
 // Let animations/transitions settle before the next observation so the
 // screenshot the LLM reasons over reflects the action's result.
 const POST_GESTURE_SETTLE_MS = 500;
@@ -61,6 +64,8 @@ export class IOSDevice {
     appPath;
     /** udid of the single booted simulator we drive. */
     udid = "";
+    /** Set once the WebDriverAgent runner is up, so the startup note logs once. */
+    wdaStarted = false;
     /** POINT size — what idb ui tap/swipe consume (de-normalization basis for TAPS). */
     pointWidth = 0;
     pointHeight = 0;
@@ -96,6 +101,14 @@ export class IOSDevice {
             this.bundleId = await this.resolveBundleId(target);
         }
         const bundleId = this.bundleId;
+        // Bring up the WebDriverAgent runner (install + simctl-launch the prebuilt
+        // xctrunner, open a session). Idempotent and reused across participants, so
+        // the ~30-60s first-launch cost is paid once per run.
+        if (!this.wdaStarted) {
+            this.log("Starting the iOS automation runner (WebDriverAgent); first launch can take ~30-60s...");
+        }
+        await ensureWda(this.udid);
+        this.wdaStarted = true;
         // Prime screen geometry (points) before the first de-normalization.
         await this.refreshScreen();
         // Per-participant reset: terminate then relaunch from a clean state.
@@ -103,6 +116,21 @@ export class IOSDevice {
         await launchApp(this.udid, bundleId);
         await settle(1500); // cold start needs longer than a gesture settle
     }
+    /**
+     * The installed app's version/build, read off the simulator after
+     * launchOrReset has resolved the bundle id. Best-effort — null until the
+     * bundle id is known, or if simctl/plutil can't report it.
+     */
+    async appBuild() {
+        if (!this.bundleId || !this.udid)
+            return null;
+        const meta = await appBuildFromSimulator(this.udid, this.bundleId);
+        return {
+            package: this.bundleId,
+            version: meta?.version ?? null,
+            build: meta?.build ?? null,
+        };
+    }
     /**
      * Resolve the bundle id to drive, returning a non-null id or throwing.
      * Installs a local `.app` first and reads its CFBundleIdentifier from
@@ -171,7 +199,7 @@ export class IOSDevice {
         };
     }
     /**
-     * Read + serialize the idb describe-all a11y tree (bounds in POINTS). Any
+     * Read + serialize WDA's /source a11y tree (bounds in POINTS). Any
      * failure (retries exhausted on a trivial tree, parse error) degrades to an
      * empty tree so the backend falls back to vision — a missing tree must never
      * abort the observation.
@@ -179,7 +207,7 @@ export class IOSDevice {
     async dumpTree() {
         try {
             const json = await describeAll(this.udid);
-            const nodes = parseIdbDescribeAll(json);
+            const nodes = parseXcuiHierarchy(json);
             const tree = serializeNativeTree(nodes);
             this.log(`a11y tree: ${tree.nodeMap.size} node(s)`);
             return tree;
@@ -206,7 +234,7 @@ export class IOSDevice {
         // separately in points (see toPoints()).
         return { width: this.pixelWidth, height: this.pixelHeight };
     }
-    /** Normalized 0-1000 → POINT space (idb ui tap/swipe take points). */
+    /** Normalized 0-1000 → POINT space (WDA taps/swipes take points). */
     toPoints(c) {
         return deNormalizePoint(c, this.pointWidth, this.pointHeight);
     }
@@ -227,7 +255,7 @@ export class IOSDevice {
             const bounds = this.lastNodeMap.get(action.node_id);
             if (!bounds)
                 return { pt: null, px: null, stale: true };
-            const pt = boundsCenter(bounds); // POINTS — idb taps directly
+            const pt = boundsCenter(bounds); // POINTS — WDA taps directly
             const px = pointToPixel(pt, this.pointWidth, this.pointHeight, this.pixelWidth, this.pixelHeight);
             return { pt, px, stale: false };
         }
@@ -237,7 +265,7 @@ export class IOSDevice {
     }
     async executeAction(action) {
         try {
-            // pt drives the idb TAP (points); px is what we RECORD (pixels). ELEMENT
+            // pt drives the WDA TAP (points); px is what we RECORD (pixels). ELEMENT
             // path: pt = bounds-center, px = that center scaled to pixels. VISION
             // path: both derive from the same normalized coord. Either way the tap
             // lands right and the recorded px round-trips against dimensions().
@@ -279,6 +307,11 @@ export class IOSDevice {
                     await this.navigateBack();
                     break;
                 }
+                case "open_system_panel": {
+                    // Element-less, like navigate_back: best-effort top-edge pull-down.
+                    await this.openSystemPanel(action.panel === "quick_settings" ? "quick_settings" : "notifications");
+                    break;
+                }
                 case "drag": {
                     // A drag GRABS an element and RELEASES it elsewhere ("click the
                     // element, move, let go") — distinct from a swipe (element-less
@@ -364,7 +397,7 @@ export class IOSDevice {
             await settle(250);
         }
         const text = resolveTextValue(action, this.contextValues);
-        // idb ui text appends to the focused field; for click_type (replace) there
+        // WDA text input appends to the focused field; for click_type (replace) there
         // is no idb "clear", so we rely on the field being empty after focus. The
         // vision agent typically taps an empty field, so this matches Android's
         // common path; a true select-all clear isn't exposed by idb.
@@ -470,10 +503,10 @@ export class IOSDevice {
      * do drive the system gesture) when no back button is visible.
      */
     async navigateBack() {
-        const nodes = parseIdbDescribeAll(await describeAll(this.udid));
+        const nodes = parseXcuiHierarchy(await describeAll(this.udid));
         const back = this.findBackButton(nodes);
         if (back) {
-            const c = boundsCenter(back.bounds); // POINTS — idb taps directly
+            const c = boundsCenter(back.bounds); // POINTS — WDA taps directly
             await uiTap(this.udid, c.x, c.y);
             return;
         }
@@ -483,6 +516,39 @@ export class IOSDevice {
         const midY = Math.round(this.pointHeight / 2);
         await uiSwipe(this.udid, 1, midY, Math.round(this.pointWidth * 0.5), midY, 300);
     }
+    /**
+     * Best-effort open of an iOS system panel by swiping down from the top edge.
+     * iOS has no `cmd statusbar` equivalent, so on a Face-ID layout:
+     *   - notifications → Notification Center: swipe down from the top-CENTER.
+     *   - quick_settings → Control Center: swipe down from the top-RIGHT corner.
+     * Coordinates are POINTS (idb consumes points; see toPoints()/the swipe()
+     * helper). This is FLAKY on the simulator — idb's synthetic touch frequently
+     * doesn't trigger the system edge gesture (the same limitation navigateBack's
+     * edge-swipe hits). We compare a before/after screenshot and log LOUDLY when
+     * the screen didn't change, rather than silently reporting success, so a
+     * no-op is visible in the run. The executeAction caller still returns
+     * success:true (the gesture was attempted); the loud log is the signal.
+     */
+    async openSystemPanel(panel) {
+        const before = await screenshotPng();
+        const w = this.pointWidth;
+        const h = this.pointHeight;
+        // Start ON the top edge and travel a third of the screen down. Control
+        // Center lives under the top-right (battery/status) corner on Face-ID
+        // devices; Notification Center under the top-center notch area.
+        const startX = panel === "quick_settings" ? Math.round(w * 0.92) : Math.round(w * 0.5);
+        const startY = 1;
+        const endY = Math.round(h * 0.35);
+        await uiSwipe(this.udid, startX, startY, startX, endY, 350);
+        await settle();
+        // Loudly surface a no-op: the simulator's synthetic touch often can't drive
+        // the system edge gesture. An identical screenshot means the panel didn't open.
+        const after = await screenshotPng();
+        if (before.equals(after)) {
+            this.log(`open_system_panel (${panel}): top-edge swipe produced no visible change — ` +
+                `the simulator's synthetic touch likely didn't trigger the system gesture (flaky on the simulator).`);
+        }
+    }
     /**
      * The nav-bar back button: the leading (leftmost) actionable button in the
      * top nav-bar band. iOS HIG guarantees "back" is the leading nav item in a
@@ -526,7 +592,7 @@ export class IOSDevice {
         // the agent can adapt.
         const hint = {
             pinch_zoom: "no multi-touch on the native driver; the agent should zoom via double_tap",
-            rotate_device: "idb exposes no clean rotate; leave orientation as-is",
+            rotate_device: "rotation is not wired on the native driver; leave orientation as-is",
             keyboard_shortcut: "no hardware keyboard on the native driver; use on-screen taps/text_input",
             switch_tab: "tabs are a browser concept; the native app has a single window",
             close_tab: "tabs are a browser concept; the native app has a single window",
@@ -540,7 +606,9 @@ export class IOSDevice {
         return "";
     }
     async close() {
-        // Leave the app installed/running; the simulator is shared and the next run
-        // resets via launchOrReset. Nothing to tear down (no IME state on iOS).
+        // Tear down the WebDriverAgent session (the runner is left installed on the
+        // shared simulator for the next run). The app resets via launchOrReset; no
+        // IME state to restore on iOS.
+        await closeWda(this.udid);
     }
 }

package/dist/lib/local-sim/loop.js

@@ -7,8 +7,9 @@
  */
 import { launchSharedBrowser, FULL_PAGE_HEIGHT_CAP_PX_MOBILE, FULL_PAGE_HEIGHT_CAP_PX_DESKTOP, } from "./browser.js";
 import { uploadScreenshot } from "./upload.js";
-import { detectNoVisibleChange, describeAction } from "./actions.js";
+import { detectNoVisibleChange, describeAction, classifyStepKind } from "./actions.js";
 import { createDevice } from "./device.js";
+import pkg from "../../../package.json" with { type: "json" };
 import { enableDebug, isDebugEnabled, debugRawResponse, debugNormalizedActions, debugActionExecution, debugForwards, debugStepSummary, debugRecord, } from "./debug.js";
 /**
  * Native (mobile) platforms drive a single physical device via screenshot →
@@ -49,6 +50,7 @@ export function flattenAction(raw, nodeId = null, nodeDescription = null) {
         key: a.key ?? null,
         tab_id: a.tab_id ?? null,
         orientation: a.orientation ?? null,
+        panel: a.panel ?? null,
         scale: a.scale ?? null,
         // Native path: ResolvedAction.coordinates (top level of the resolved_actions
         // entry) is the single {x,y} execution point. Fall back to the nested action
@@ -112,6 +114,34 @@ const SENTIMENT_ICONS = {
     Positive: "+", Negative: "-", Neutral: "~",
     Frustrated: "!", Confused: "?", Delighted: "*",
 };
+const CLI_VERSION = pkg.version;
+/**
+ * Stamp the app build this run drove onto the iteration, so the web app's
+ * run-settings card can show which build the iteration is on. Best-effort:
+ * a native run never depends on this landing, so failures are warned, not
+ * thrown. Only native platforms carry a build.
+ */
+async function reportObservedApp(client, iterationId, platform, build, log) {
+    if (platform !== "ios" && platform !== "android")
+        return;
+    try {
+        await client.post(`/iterations/${iterationId}/observed-app`, {
+            platform,
+            package: build.package,
+            version: build.version,
+            build: build.build,
+            cli_version: CLI_VERSION,
+        });
+        const label = [build.version, build.build ? `(${build.build})` : null]
+            .filter(Boolean)
+            .join(" ");
+        log(`Recorded app build${label ? `: ${label}` : ""}`);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.warn(`Could not record app build for the iteration: ${msg}`);
+    }
+}
 /**
  * Run local simulations — parallel when multiple participants, sequential by default.
  * Use --parallel <n> to control concurrency (default: number of participants).
@@ -138,6 +168,16 @@ export async function runLocalSimulations(client, opts) {
         log("Native (android/ios) runs drive a single device — running sequentially.");
     }
     const concurrency = isNativeRun ? 1 : (opts.parallel ?? opts.participantIds.length);
+    // Native runs stamp the app build onto the iteration once — every
+    // participant in a run drives the same installed build, so dedupe to a
+    // single best-effort POST after the first device resolves its app.
+    let appBuildReported = false;
+    const reportAppBuild = (build, platform) => {
+        if (appBuildReported)
+            return;
+        appBuildReported = true;
+        void reportObservedApp(client, opts.iterationId, platform, build, log);
+    };
     try {
         if (concurrency <= 1 || opts.participantIds.length <= 1) {
             // Sequential execution — each participant owns its own browser
@@ -148,7 +188,7 @@ export async function runLocalSimulations(client, opts) {
                 log(`\nStarting local simulation for ${participantName}...`);
                 try {
                     const participantLog = (msg) => log(`[${participantName}] ${msg}`);
-                    await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled);
+                    await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled, reportAppBuild);
                     log(`Completed: ${participantName}`);
                 }
                 catch (err) {
@@ -182,7 +222,7 @@ export async function runLocalSimulations(client, opts) {
                         const participantLog = (msg) => log(`[${participantName}] ${msg}`);
                         participantLog("Starting...");
                         try {
-                            await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled, sharedBrowser);
+                            await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled, reportAppBuild, sharedBrowser);
                             participantLog("Completed");
                         }
                         catch (err) {
@@ -202,7 +242,7 @@ export async function runLocalSimulations(client, opts) {
         process.off("SIGINT", onSigint);
     }
 }
-async function runSingleSimulation(client, participantId, participantName, opts, log, isCancelled, sharedBrowser) {
+async function runSingleSimulation(client, participantId, participantName, opts, log, isCancelled, onAppBuild, sharedBrowser) {
     // Step 1: Initialize session
     const initResponse = await client.localSimInit({
         participant_id: participantId,
@@ -273,6 +313,19 @@ async function runSingleSimulation(client, participantId, participantName, opts,
     try {
         // Step 3: Launch / navigate the target to its starting point.
         await device.launchOrReset(launchTarget);
+        // Step 3b: Capture the installed app's build (native only). Best-effort —
+        // the dedupe in runLocalSimulations keeps this to one POST per run, and a
+        // failed read or report never disturbs the simulation.
+        if (onAppBuild) {
+            try {
+                const observed = await device.appBuild?.();
+                if (observed)
+                    onAppBuild(observed, platform);
+            }
+            catch {
+                // ignore — build capture is non-essential
+            }
+        }
         // Step 4: Run assignment loop
         for (let assignmentIdx = 0; assignmentIdx < session.assignments.length; assignmentIdx++) {
             const assignment = session.assignments[assignmentIdx];
@@ -283,6 +336,12 @@ async function runSingleSimulation(client, participantId, participantName, opts,
             // status when the loop ends because the agent terminated (completed vs
             // abandoned). Stays "in_progress" if the loop hits max_steps.
             let lastAssignmentStatus = "in_progress";
+            // Frame continuity (native): carry the PREVIOUS step's logical-screen
+            // classification + matched frame forward, so this step's match-frame call
+            // can tell the backend to reuse the frame when the screen didn't change
+            // (pure scroll / non-submitting keyboard). Reset per assignment.
+            let lastStepKind = "none";
+            let lastFrameVersionId;
             while (step < maxSteps && !assignmentCompleted && !isCancelled()) {
                 // OBSERVE — the device refreshes its own active surface (popup /
                 // switch_tab for browser) before capturing. (The browser device emits
@@ -394,12 +453,16 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                 const actionDescs = [];
                 const elementNames = [];
                 const actionDebugEntries = [];
+                // Per-action success (index-aligned with stepResponse.actions), used to
+                // classify this step's logical-screen kind for frame continuity.
+                const perActionSuccess = [];
                 const preActionScreenshot = await device.captureScreenshot();
                 for (let i = 0; i < stepResponse.actions.length; i++) {
                     if (isCancelled())
                         break;
                     const action = stepResponse.actions[i];
                     const result = await device.executeAction(action);
+                    perActionSuccess[i] = result.success;
                     const desc = describeAction(action);
                     debugActionExecution(i, action, result, action.node_id ? "cdp" : "playwright");
                     const openedNewTab = result.openedNewTab;
@@ -414,28 +477,44 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     const actionType = action.type || "unknown";
                     const INTERNAL_ACTIONS = new Set(["think"]);
                     if (!INTERNAL_ACTIONS.has(actionType)) {
+                        // Pack `data` to match the hosted sim's map_action_to_db so native
+                        // rows render identically. value_type lets the FE flag var/secret;
+                        // drag's full path goes under data.coordinates (0-1000), not a
+                        // bespoke drag_end. Secret `value` stays masked (it's the variable
+                        // key, not the resolved secret — masking is strictly safer than the
+                        // web path, and value_type now drives the FE lock glyph).
+                        const actionData = {
+                            ...(action.value !== undefined && action.value !== null && { value: action.value_type === "secret" ? "***" : action.value }),
+                            ...(action.value_type && { value_type: action.value_type }),
+                            ...(action.mode && { mode: action.mode }),
+                            ...(action.submit && { submit: action.submit }),
+                            ...(action.direction && { direction: action.direction }),
+                            ...(action.amount && { amount: action.amount }),
+                            ...(action.count && action.count > 1 && { count: action.count }),
+                            ...(action.duration_ms && { duration_ms: action.duration_ms }),
+                            ...(action.modifiers?.length && { modifiers: action.modifiers }),
+                            ...(action.key && { key: action.key }),
+                            ...(action.tab_id && { tab_id: action.tab_id }),
+                            ...(action.orientation && { orientation: action.orientation }),
+                            ...(action.panel && { panel: action.panel }),
+                            ...(action.drag && {
+                                coordinates: {
+                                    startX: action.drag.startX,
+                                    startY: action.drag.startY,
+                                    endX: action.drag.endX,
+                                    endY: action.drag.endY,
+                                },
+                            }),
+                            ...(openedNewTab && { opened_new_tab: true }),
+                        };
                         actionDatas.push({
                             action_type: actionType,
                             element_label: action.element_name ?? null,
                             element_type: action.element_type ?? null,
-                            coordinates: normalizedCoords,
-                            data: {
-                                ...(action.value !== undefined && action.value !== null && { value: action.value_type === "secret" ? "***" : action.value }),
-                                ...(action.mode && { mode: action.mode }),
-                                ...(action.submit && { submit: action.submit }),
-                                ...(action.direction && { direction: action.direction }),
-                                ...(action.amount && { amount: action.amount }),
-                                ...(action.count && action.count > 1 && { count: action.count }),
-                                ...(action.duration_ms && { duration_ms: action.duration_ms }),
-                                ...(action.modifiers?.length && { modifiers: action.modifiers }),
-                                ...(action.key && { key: action.key }),
-                                ...(action.tab_id && { tab_id: action.tab_id }),
-                                ...(action.orientation && { orientation: action.orientation }),
-                                // The recorded `coordinates` is the drag START; persist the END
-                                // (normalized 0-1000) too so the journey captures the full path.
-                                ...(action.drag && { drag_end: { x: action.drag.endX, y: action.drag.endY } }),
-                                ...(openedNewTab && { opened_new_tab: true }),
-                            },
+                            // Drag's path lives in data.coordinates; the hosted sim leaves the
+                            // top-level coordinates null for a drag.
+                            coordinates: action.drag ? null : normalizedCoords,
+                            data: Object.keys(actionData).length ? actionData : null,
                             order: i,
                         });
                     }
@@ -492,6 +571,15 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                         // Native: drive FrameSourceType.ANDROID/IOS directly; browser falls
                         // back to screen_format server-side.
                         platform,
+                        // Frame continuity: these describe the transition INTO this
+                        // observation, produced by the PREVIOUS step's action. When that
+                        // step was a pure scroll / non-submitting keyboard on a native
+                        // device, the logical screen didn't change — tell the backend to
+                        // reuse the previous frame instead of minting a new one off the
+                        // shifted pixels. Carried from lastStepKind / lastFrameVersionId,
+                        // updated AFTER this call for the next iteration.
+                        ...(isNative && lastFrameVersionId ? { previous_frame_version_id: lastFrameVersionId } : {}),
+                        same_screen_continuation: isNative && (lastStepKind === "scroll" || lastStepKind === "keyboard"),
                     });
                     frameVersionId = matchResult.frame_version_id;
                 }
@@ -499,6 +587,11 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     const msg = err instanceof Error ? err.message : String(err);
                     log(`    Warning: frame matching failed — ${msg}`);
                 }
+                // Carry THIS step's logical-screen classification + matched frame
+                // forward for the NEXT iteration's match-frame call (consumed above as
+                // last*). Classify after the call so ordering is consume-then-update.
+                lastStepKind = classifyStepKind(stepResponse.actions, perActionSuccess);
+                lastFrameVersionId = frameVersionId;
                 // Debug-only: capture post-action screenshot to show result
                 let postActionBase64;
                 if (isDebugEnabled()) {
@@ -518,7 +611,7 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     forwards.push({ type: "LOOP_DETECTED", content: "A repetitive action cycle was detected. Try a different approach." });
                 }
                 // Record interaction (1-indexed step for backend)
-                interactions.push({
+                const interaction = {
                     step: step + 1,
                     assignment_id: assignment.id,
                     ...(screenshotUrl ? { screenshot_url: screenshotUrl } : { screenshot_base64: obsBase64 }),
@@ -542,7 +635,24 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     // Server reduces this to Interaction.tab when N >= 2; omit on
                     // single-tab steps to keep the payload (and DB column) null.
                     ...(tabsSnapshot.length >= 2 ? { tabs: tabsSnapshot } : {}),
-                });
+                };
+                // Keep the in-memory array for the debug HTML report.
+                interactions.push(interaction);
+                // Stream this interaction live so the backend persists + commits it
+                // immediately and fires INTERACTION_CREATED in realtime. A streaming
+                // failure must never abort the run — log and continue (the run-end
+                // finalize call still records the terminal state).
+                try {
+                    await client.localSimRecordInteraction({
+                        participant_id: session.participant_id,
+                        product_id: session.product_id,
+                        interaction,
+                    });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    log(`  Warning: failed to stream interaction ${step + 1} — ${msg}`);
+                }
                 // Update history for next step
                 history.push({
                     comment: stepResponse.comment,
@@ -633,7 +743,6 @@ async function runSingleSimulation(client, participantId, participantName, opts,
             await client.localSimRecord({
                 participant_id: session.participant_id,
                 product_id: session.product_id,
-                interactions,
                 final_status: finalStatus,
                 assignment_statuses: assignmentStatuses,
             });

package/dist/lib/local-sim/native-a11y.d.ts

@@ -12,7 +12,7 @@
  *
  * COORDINATE SPACE — carried, not converted, by this module:
  *   - Android `uiautomator dump` bounds are screencap PIXELS (`space: "px"`).
- *   - iOS `idb ui describe-all` frames are POINTS (`space: "points"`).
+ *   - iOS WebDriverAgent /source frames are POINTS (`space: "points"`).
  * The device de-normalizes/taps in its own space (AndroidDevice taps pixels;
  * IOSDevice taps points), so the `space` tag tells the caller which dimension a
  * node's bounds-center belongs to. This module never mixes the two.
@@ -65,15 +65,29 @@ export interface NativeTree {
  */
 export declare function parseUiautomatorXml(xml: string): NativeNode[];
 /**
- * Parse `idb ui describe-all` JSON (a FLAT array of elements, each with a `frame`
- * in POINTS) into NativeNodes in array order. iOS is already a flat,
- * properly-labeled list — no ancestor walk needed — so `clickable` is derived
- * from the element's role/type and whether it carries a usable label.
+ * Parse WDA's `GET /source?format=json` — a NESTED accessibility tree — into the
+ * FLAT, depth-first `NativeNode[]` (POINTS) that `parseXcuiHierarchy` produces,
+ * so `serializeNativeTree` consumes it unchanged. WDA's `type` matches idb's iOS
+ * types (Button/StaticText/SearchField/Cell/Image/Application…), so
+ * `normalizeRole`/`IOS_ACTIONABLE_TYPES`/`frameToBounds` all apply as-is.
+ *
+ * KEY: WDA's `/source` is the FULL XCUIElement tree — every container and leaf —
+ * NOT idb's clean accessibility-elements list. iOS settings rows surface as an
+ * accessible `Button` ("General", isAccessible=1) that ALSO contains a duplicate
+ * inner `StaticText` ("General", isAccessible=0) and is wrapped in a `Cell`
+ * (isAccessible=0). Emitting all three yields "General General" + empty
+ * listitems. So we emit ONLY `isAccessible && isVisible` nodes — exactly the
+ * VoiceOver-exposed set idb returned: the labeled Button is both the label and
+ * the tap target; the duplicate StaticText and the wrapping Cell are pruned. A
+ * sparse a11y tree degrades to the loop's vision fallback, so strict filtering
+ * never strands the run.
+ *
+ * Accepts either the raw tree or the W3C `{ value: <tree> }` envelope WDA returns.
  */
-export declare function parseIdbDescribeAll(json: string): NativeNode[];
+export declare function parseXcuiHierarchy(json: string): NativeNode[];
 /**
  * Serialize a flat NativeNode list (from `parseUiautomatorXml` /
- * `parseIdbDescribeAll`) into the `[id] role "label"` string the DOMLocator
+ * `parseXcuiHierarchy`) into the `[id] role "label"` string the DOMLocator
  * reasons over, plus a `shortId → bounds` map for local tap resolution.
  *
  * Emission rules (kept tight, like the DOM serializer):