@ishlabs/cli 0.26.0 → 0.26.1

package/dist/lib/local-sim/adb.js

@@ -10,30 +10,90 @@
  * backend's 0-1000 coordinates against the screencap pixel size and taps
  * directly. (Verified by the Layer-1 driver smoke; see scripts/mobile-e2e.)
  */
-import { execFile } from "node:child_process";
-import { existsSync } from "node:fs";
+import { execFile, execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
 import { promisify } from "node:util";
+import { binDir, adbBin } from "../paths.js";
 const execFileAsync = promisify(execFile);
-// adb ships with Homebrew's android-platform-tools and inside the SDK. Prefer
-// an explicit absolute path so we never depend on the caller's PATH (mirrors
-// scripts/mobile-e2e/lib.sh). Override with ISH_ADB / ADB.
-function resolveAdb() {
+// Resolve adb without depending on the caller's PATH: ISH_ADB/ADB override → the
+// Android SDK → Homebrew → our own download cache → PATH. If none is found,
+// ensureAdb() fetches Google's standalone platform-tools (a small zip) into
+// ~/.ish/bin, mirroring how cloudflared / the iOS WebDriverAgent runner are
+// fetched. Override the binary with ISH_ADB / ADB.
+function findAdb() {
     const fromEnv = process.env.ISH_ADB || process.env.ADB;
     if (fromEnv && existsSync(fromEnv))
         return fromEnv;
-    const homebrew = "/opt/homebrew/bin/adb";
-    if (existsSync(homebrew))
-        return homebrew;
     const sdkHome = process.env.ANDROID_HOME || process.env.ANDROID_SDK_ROOT;
     if (sdkHome) {
-        const sdkAdb = `${sdkHome}/platform-tools/adb`;
+        const sdkAdb = join(sdkHome, "platform-tools", "adb");
         if (existsSync(sdkAdb))
             return sdkAdb;
     }
-    // Last resort: rely on PATH and surface a clear error if it's missing.
-    return "adb";
+    const homebrew = "/opt/homebrew/bin/adb";
+    if (existsSync(homebrew))
+        return homebrew;
+    if (existsSync(adbBin()))
+        return adbBin(); // our downloaded cache
+    // PATH fallback — only if `adb` actually resolves there.
+    try {
+        execFileSync(process.platform === "win32" ? "where" : "which", ["adb"], { stdio: "ignore" });
+        return "adb";
+    }
+    catch {
+        return null;
+    }
+}
+let cachedAdb = null;
+/** Resolve adb, downloading Google's platform-tools on first use if not found. */
+export async function ensureAdb() {
+    if (cachedAdb)
+        return cachedAdb;
+    cachedAdb = findAdb() ?? (await downloadAdb());
+    return cachedAdb;
+}
+const PLATFORM_TOOLS_OS = {
+    darwin: "darwin",
+    linux: "linux",
+    win32: "windows",
+};
+/** Fetch + unpack Google's standalone platform-tools into ~/.ish/bin. */
+async function downloadAdb() {
+    const os = PLATFORM_TOOLS_OS[process.platform];
+    if (!os) {
+        throw new AdbError(`no prebuilt adb for ${process.platform}; install Android platform-tools and set ISH_ADB`);
+    }
+    const url = `https://dl.google.com/android/repository/platform-tools-latest-${os}.zip`;
+    const dir = binDir();
+    console.error("Fetching adb (Android platform-tools) from Google...");
+    mkdirSync(dir, { recursive: true });
+    const zipPath = join(dir, "platform-tools.zip");
+    let resp;
+    try {
+        resp = await fetch(url, { signal: AbortSignal.timeout(120_000) });
+    }
+    catch (e) {
+        throw new AdbError(`failed to download platform-tools from ${url}: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    if (!resp.ok)
+        throw new AdbError(`failed to download platform-tools: HTTP ${resp.status} from ${url}`);
+    writeFileSync(zipPath, Buffer.from(await resp.arrayBuffer()));
+    try {
+        // The zip carries a top-level `platform-tools/` dir; extract into binDir().
+        const [cmd, args] = process.platform === "win32"
+            ? ["tar", ["-xf", zipPath, "-C", dir]]
+            : ["unzip", ["-o", "-q", zipPath, "-d", dir]];
+        await execFileAsync(cmd, args, { timeout: 120_000 });
+    }
+    catch (e) {
+        throw new AdbError(`failed to unpack platform-tools: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    rmSync(zipPath, { force: true });
+    if (!existsSync(adbBin()))
+        throw new AdbError(`platform-tools unpacked but adb is missing at ${adbBin()}`);
+    return adbBin();
 }
-const ADB = resolveAdb();
 const DEFAULT_TIMEOUT_MS = 30_000;
 // screencap on a cold emulator frame can be slow; give it generous headroom.
 const SCREENCAP_TIMEOUT_MS = 30_000;
@@ -47,8 +107,9 @@ export class AdbError extends Error {
 }
 /** Run `adb <args>` and return trimmed stdout. Throws AdbError on failure. */
 export async function adb(args, timeoutMs = DEFAULT_TIMEOUT_MS) {
+    const bin = await ensureAdb();
     try {
-        const { stdout } = await execFileAsync(ADB, args, {
+        const { stdout } = await execFileAsync(bin, args, {
             timeout: timeoutMs,
             maxBuffer: 4 * 1024 * 1024,
         });
@@ -63,14 +124,43 @@ export async function adb(args, timeoutMs = DEFAULT_TIMEOUT_MS) {
 export async function adbShell(args, timeoutMs = DEFAULT_TIMEOUT_MS) {
     return adb(["shell", ...args], timeoutMs);
 }
+/**
+ * Pull versionName / versionCode out of `dumpsys package <pkg>` text. The
+ * relevant lines read `versionCode=42 minSdk=24 targetSdk=34` and
+ * `versionName=1.2.3`; `\d+` stops the build before the trailing tokens and
+ * `\S+` takes the version up to the next space. Returns null when neither is
+ * present (wrong/empty package).
+ */
+export function parseDumpsysAppBuild(out) {
+    const version = out.match(/versionName=(\S+)/)?.[1] ?? null;
+    const build = out.match(/versionCode=(\d+)/)?.[1] ?? null;
+    if (!version && !build)
+        return null;
+    return { version, build };
+}
+/**
+ * Read an installed package's versionName / versionCode from
+ * `dumpsys package <pkg>`. Best-effort: returns null on any failure (the run
+ * never depends on it). Covers both freshly-installed apks and pre-installed
+ * packages — by call time the package name is already resolved.
+ */
+export async function appBuildFromDevice(pkg) {
+    try {
+        return parseDumpsysAppBuild(await adbShell(["dumpsys", "package", pkg], 30_000));
+    }
+    catch {
+        return null;
+    }
+}
 /**
  * Capture the current screen as raw PNG bytes via `adb exec-out screencap -p`.
  * `exec-out` (not `shell`) avoids the CRLF translation that corrupts binary
  * output. Returns the PNG buffer at full device resolution.
  */
 export async function screencapPng() {
+    const bin = await ensureAdb();
     try {
-        const { stdout } = await execFileAsync(ADB, ["exec-out", "screencap", "-p"], {
+        const { stdout } = await execFileAsync(bin, ["exec-out", "screencap", "-p"], {
             timeout: SCREENCAP_TIMEOUT_MS,
             maxBuffer: SCREENCAP_MAX_BUFFER,
             encoding: "buffer",
@@ -90,7 +180,7 @@ export async function requireOneDevice() {
     }
     catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
-        throw new AdbError(`Could not run adb (looked for "${ADB}"). Run \`ish check android\` to check your setup. ${msg}`);
+        throw new AdbError(`Could not run adb (looked for "${findAdb() ?? "adb"}"). Run \`ish check android\` to check your setup. ${msg}`);
     }
     // Output: "List of devices attached\n<serial>\tdevice\n..."
     const online = out

package/dist/lib/local-sim/android.d.ts

@@ -19,7 +19,7 @@
  *   - Vision path: px = round(x / 1000 * screencapWidth); same for y.
  */
 import type { LocalStepAction, ContextValue } from "./types.js";
-import type { SimulationDevice, DeviceObservation, DeviceActionResult } from "./device.js";
+import type { SimulationDevice, DeviceObservation, DeviceActionResult, AppBuild } from "./device.js";
 export interface AndroidDeviceOptions {
     /** App package name to force-stop/relaunch between participants. May be derived from --app. */
     appPackage?: string;
@@ -47,6 +47,12 @@ export declare class AndroidDevice implements SimulationDevice {
     private adbKeyboardActive;
     constructor(opts: AndroidDeviceOptions);
     launchOrReset(target: string): Promise<void>;
+    /**
+     * The installed app's version/build, read off the device after
+     * launchOrReset has resolved the package. Best-effort — null until the
+     * package is known, or if dumpsys can't report it.
+     */
+    appBuild(): Promise<AppBuild | null>;
     /**
      * Resolve which package to drive, returning a non-null package name or
      * throwing. For a local .apk we read the package straight from its binary

package/dist/lib/local-sim/android.js

@@ -19,7 +19,7 @@
  *   - Vision path: px = round(x / 1000 * screencapWidth); same for y.
  */
 import { resolveTextValue } from "./actions.js";
-import { requireOneDevice, screencapPng, pngDimensions, dumpUiautomatorXml, inputTap, inputSwipe, inputDrag, inputLongPress, setUserRotation, forceStop, launchApp, installApk, isPackageInstalled, listPackages, isAdbKeyboardInstalled, enableAdbKeyboard, setIme, resetIme, currentIme, adbKeyboardType, adbKeyboardClear, pressKeyEvent, statusbarExpand, ADB_KEYBOARD_PKG, } from "./adb.js";
+import { requireOneDevice, screencapPng, pngDimensions, dumpUiautomatorXml, inputTap, inputSwipe, inputDrag, inputLongPress, setUserRotation, forceStop, launchApp, installApk, isPackageInstalled, listPackages, isAdbKeyboardInstalled, enableAdbKeyboard, setIme, resetIme, currentIme, adbKeyboardType, adbKeyboardClear, pressKeyEvent, statusbarExpand, appBuildFromDevice, ADB_KEYBOARD_PKG, } from "./adb.js";
 import { isLocalPath } from "../upload.js";
 import { deNormalizePoint, deNormalizeDrag } from "./coordinates.js";
 import { parseUiautomatorXml, serializeNativeTree, boundsCenter } from "./native-a11y.js";
@@ -74,6 +74,21 @@ export class AndroidDevice {
         // Prime screencap dimensions for the first de-normalization.
         await this.refreshDimensions();
     }
+    /**
+     * The installed app's version/build, read off the device after
+     * launchOrReset has resolved the package. Best-effort — null until the
+     * package is known, or if dumpsys can't report it.
+     */
+    async appBuild() {
+        if (!this.appPackage)
+            return null;
+        const meta = await appBuildFromDevice(this.appPackage);
+        return {
+            package: this.appPackage,
+            version: meta?.version ?? null,
+            build: meta?.build ?? null,
+        };
+    }
     /**
      * Resolve which package to drive, returning a non-null package name or
      * throwing. For a local .apk we read the package straight from its binary

package/dist/lib/local-sim/device.d.ts

@@ -56,6 +56,19 @@ export interface DeviceActionResult {
     } | null;
     openedNewTab: boolean;
 }
+/**
+ * The version/build of the installed native app being driven, read off the
+ * device after `launchOrReset`. Lets the web app show which build an iteration
+ * last ran against. `package` is the resolved bundle id (iOS) / package name
+ * (Android); `version` is the marketing version (CFBundleShortVersionString /
+ * versionName) and `build` the build number (CFBundleVersion / versionCode),
+ * either of which may be null when the device doesn't report it.
+ */
+export interface AppBuild {
+    package: string;
+    version: string | null;
+    build: string | null;
+}
 /**
  * A drivable simulation target. Implementations own their own lifecycle and
  * (for the browser) tab bookkeeping.
@@ -90,6 +103,12 @@ export interface SimulationDevice {
     executeAction(action: LocalStepAction): Promise<DeviceActionResult>;
     /** Current location string for recording (URL for browser; "" for native). */
     currentUrl(): string;
+    /**
+     * Native only: the version/build of the installed app being driven, read
+     * off the device after `launchOrReset`. Browser omits it. Best-effort — a
+     * failed read resolves to null and never disturbs the run.
+     */
+    appBuild?(): Promise<AppBuild | null>;
     /** Tear down. For shared-browser tabs this closes just the tab. */
     close(): Promise<void>;
 }

package/dist/lib/local-sim/ios.d.ts

@@ -31,7 +31,7 @@
  *   backend never converts coords with screen_width/height.
  */
 import type { LocalStepAction, ContextValue } from "./types.js";
-import type { SimulationDevice, DeviceObservation, DeviceActionResult } from "./device.js";
+import type { SimulationDevice, DeviceObservation, DeviceActionResult, AppBuild } from "./device.js";
 export interface IosDeviceOptions {
     /** Bundle id to terminate/relaunch between participants. Derived from --app when a .app is given. */
     bundleId?: string;
@@ -70,6 +70,12 @@ export declare class IOSDevice implements SimulationDevice {
     private lastNodeMap;
     constructor(opts: IosDeviceOptions);
     launchOrReset(target: string): Promise<void>;
+    /**
+     * The installed app's version/build, read off the simulator after
+     * launchOrReset has resolved the bundle id. Best-effort — null until the
+     * bundle id is known, or if simctl/plutil can't report it.
+     */
+    appBuild(): Promise<AppBuild | null>;
     /**
      * Resolve the bundle id to drive, returning a non-null id or throwing.
      * Installs a local `.app` first and reads its CFBundleIdentifier from

package/dist/lib/local-sim/ios.js

@@ -31,7 +31,7 @@
  *   backend never converts coords with screen_width/height.
  */
 import { resolveTextValue } from "./actions.js";
-import { requireOneBootedSimulator, screenshotPng, terminateApp, launchApp, installApp, isAppInstalled, bundleIdFromApp, } from "./simctl.js";
+import { requireOneBootedSimulator, screenshotPng, terminateApp, launchApp, installApp, isAppInstalled, bundleIdFromApp, appBuildFromSimulator, } from "./simctl.js";
 // iOS UI interaction + a11y run through WebDriverAgent (XCUITest), not idb.
 import { ensureWda, closeWda, describeScreen, describeAll, uiTap, uiLongPress, uiSwipe, uiText, uiKey, HID_KEY_RETURN, } from "./xcuitest.js";
 import { isLocalPath } from "../upload.js";
@@ -116,6 +116,21 @@ export class IOSDevice {
         await launchApp(this.udid, bundleId);
         await settle(1500); // cold start needs longer than a gesture settle
     }
+    /**
+     * The installed app's version/build, read off the simulator after
+     * launchOrReset has resolved the bundle id. Best-effort — null until the
+     * bundle id is known, or if simctl/plutil can't report it.
+     */
+    async appBuild() {
+        if (!this.bundleId || !this.udid)
+            return null;
+        const meta = await appBuildFromSimulator(this.udid, this.bundleId);
+        return {
+            package: this.bundleId,
+            version: meta?.version ?? null,
+            build: meta?.build ?? null,
+        };
+    }
     /**
      * Resolve the bundle id to drive, returning a non-null id or throwing.
      * Installs a local `.app` first and reads its CFBundleIdentifier from

package/dist/lib/local-sim/loop.js

@@ -7,8 +7,9 @@
  */
 import { launchSharedBrowser, FULL_PAGE_HEIGHT_CAP_PX_MOBILE, FULL_PAGE_HEIGHT_CAP_PX_DESKTOP, } from "./browser.js";
 import { uploadScreenshot } from "./upload.js";
-import { detectNoVisibleChange, describeAction } from "./actions.js";
+import { detectNoVisibleChange, describeAction, classifyStepKind } from "./actions.js";
 import { createDevice } from "./device.js";
+import pkg from "../../../package.json" with { type: "json" };
 import { enableDebug, isDebugEnabled, debugRawResponse, debugNormalizedActions, debugActionExecution, debugForwards, debugStepSummary, debugRecord, } from "./debug.js";
 /**
  * Native (mobile) platforms drive a single physical device via screenshot →
@@ -113,6 +114,34 @@ const SENTIMENT_ICONS = {
     Positive: "+", Negative: "-", Neutral: "~",
     Frustrated: "!", Confused: "?", Delighted: "*",
 };
+const CLI_VERSION = pkg.version;
+/**
+ * Stamp the app build this run drove onto the iteration, so the web app's
+ * run-settings card can show which build the iteration is on. Best-effort:
+ * a native run never depends on this landing, so failures are warned, not
+ * thrown. Only native platforms carry a build.
+ */
+async function reportObservedApp(client, iterationId, platform, build, log) {
+    if (platform !== "ios" && platform !== "android")
+        return;
+    try {
+        await client.post(`/iterations/${iterationId}/observed-app`, {
+            platform,
+            package: build.package,
+            version: build.version,
+            build: build.build,
+            cli_version: CLI_VERSION,
+        });
+        const label = [build.version, build.build ? `(${build.build})` : null]
+            .filter(Boolean)
+            .join(" ");
+        log(`Recorded app build${label ? `: ${label}` : ""}`);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.warn(`Could not record app build for the iteration: ${msg}`);
+    }
+}
 /**
  * Run local simulations — parallel when multiple participants, sequential by default.
  * Use --parallel <n> to control concurrency (default: number of participants).
@@ -139,6 +168,16 @@ export async function runLocalSimulations(client, opts) {
         log("Native (android/ios) runs drive a single device — running sequentially.");
     }
     const concurrency = isNativeRun ? 1 : (opts.parallel ?? opts.participantIds.length);
+    // Native runs stamp the app build onto the iteration once — every
+    // participant in a run drives the same installed build, so dedupe to a
+    // single best-effort POST after the first device resolves its app.
+    let appBuildReported = false;
+    const reportAppBuild = (build, platform) => {
+        if (appBuildReported)
+            return;
+        appBuildReported = true;
+        void reportObservedApp(client, opts.iterationId, platform, build, log);
+    };
     try {
         if (concurrency <= 1 || opts.participantIds.length <= 1) {
             // Sequential execution — each participant owns its own browser
@@ -149,7 +188,7 @@ export async function runLocalSimulations(client, opts) {
                 log(`\nStarting local simulation for ${participantName}...`);
                 try {
                     const participantLog = (msg) => log(`[${participantName}] ${msg}`);
-                    await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled);
+                    await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled, reportAppBuild);
                     log(`Completed: ${participantName}`);
                 }
                 catch (err) {
@@ -183,7 +222,7 @@ export async function runLocalSimulations(client, opts) {
                         const participantLog = (msg) => log(`[${participantName}] ${msg}`);
                         participantLog("Starting...");
                         try {
-                            await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled, sharedBrowser);
+                            await runSingleSimulation(client, participantId, participantName, opts, participantLog, () => cancelled, reportAppBuild, sharedBrowser);
                             participantLog("Completed");
                         }
                         catch (err) {
@@ -203,7 +242,7 @@ export async function runLocalSimulations(client, opts) {
         process.off("SIGINT", onSigint);
     }
 }
-async function runSingleSimulation(client, participantId, participantName, opts, log, isCancelled, sharedBrowser) {
+async function runSingleSimulation(client, participantId, participantName, opts, log, isCancelled, onAppBuild, sharedBrowser) {
     // Step 1: Initialize session
     const initResponse = await client.localSimInit({
         participant_id: participantId,
@@ -274,6 +313,19 @@ async function runSingleSimulation(client, participantId, participantName, opts,
     try {
         // Step 3: Launch / navigate the target to its starting point.
         await device.launchOrReset(launchTarget);
+        // Step 3b: Capture the installed app's build (native only). Best-effort —
+        // the dedupe in runLocalSimulations keeps this to one POST per run, and a
+        // failed read or report never disturbs the simulation.
+        if (onAppBuild) {
+            try {
+                const observed = await device.appBuild?.();
+                if (observed)
+                    onAppBuild(observed, platform);
+            }
+            catch {
+                // ignore — build capture is non-essential
+            }
+        }
         // Step 4: Run assignment loop
         for (let assignmentIdx = 0; assignmentIdx < session.assignments.length; assignmentIdx++) {
             const assignment = session.assignments[assignmentIdx];
@@ -284,6 +336,12 @@ async function runSingleSimulation(client, participantId, participantName, opts,
             // status when the loop ends because the agent terminated (completed vs
             // abandoned). Stays "in_progress" if the loop hits max_steps.
             let lastAssignmentStatus = "in_progress";
+            // Frame continuity (native): carry the PREVIOUS step's logical-screen
+            // classification + matched frame forward, so this step's match-frame call
+            // can tell the backend to reuse the frame when the screen didn't change
+            // (pure scroll / non-submitting keyboard). Reset per assignment.
+            let lastStepKind = "none";
+            let lastFrameVersionId;
             while (step < maxSteps && !assignmentCompleted && !isCancelled()) {
                 // OBSERVE — the device refreshes its own active surface (popup /
                 // switch_tab for browser) before capturing. (The browser device emits
@@ -395,12 +453,16 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                 const actionDescs = [];
                 const elementNames = [];
                 const actionDebugEntries = [];
+                // Per-action success (index-aligned with stepResponse.actions), used to
+                // classify this step's logical-screen kind for frame continuity.
+                const perActionSuccess = [];
                 const preActionScreenshot = await device.captureScreenshot();
                 for (let i = 0; i < stepResponse.actions.length; i++) {
                     if (isCancelled())
                         break;
                     const action = stepResponse.actions[i];
                     const result = await device.executeAction(action);
+                    perActionSuccess[i] = result.success;
                     const desc = describeAction(action);
                     debugActionExecution(i, action, result, action.node_id ? "cdp" : "playwright");
                     const openedNewTab = result.openedNewTab;
@@ -415,29 +477,44 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     const actionType = action.type || "unknown";
                     const INTERNAL_ACTIONS = new Set(["think"]);
                     if (!INTERNAL_ACTIONS.has(actionType)) {
+                        // Pack `data` to match the hosted sim's map_action_to_db so native
+                        // rows render identically. value_type lets the FE flag var/secret;
+                        // drag's full path goes under data.coordinates (0-1000), not a
+                        // bespoke drag_end. Secret `value` stays masked (it's the variable
+                        // key, not the resolved secret — masking is strictly safer than the
+                        // web path, and value_type now drives the FE lock glyph).
+                        const actionData = {
+                            ...(action.value !== undefined && action.value !== null && { value: action.value_type === "secret" ? "***" : action.value }),
+                            ...(action.value_type && { value_type: action.value_type }),
+                            ...(action.mode && { mode: action.mode }),
+                            ...(action.submit && { submit: action.submit }),
+                            ...(action.direction && { direction: action.direction }),
+                            ...(action.amount && { amount: action.amount }),
+                            ...(action.count && action.count > 1 && { count: action.count }),
+                            ...(action.duration_ms && { duration_ms: action.duration_ms }),
+                            ...(action.modifiers?.length && { modifiers: action.modifiers }),
+                            ...(action.key && { key: action.key }),
+                            ...(action.tab_id && { tab_id: action.tab_id }),
+                            ...(action.orientation && { orientation: action.orientation }),
+                            ...(action.panel && { panel: action.panel }),
+                            ...(action.drag && {
+                                coordinates: {
+                                    startX: action.drag.startX,
+                                    startY: action.drag.startY,
+                                    endX: action.drag.endX,
+                                    endY: action.drag.endY,
+                                },
+                            }),
+                            ...(openedNewTab && { opened_new_tab: true }),
+                        };
                         actionDatas.push({
                             action_type: actionType,
                             element_label: action.element_name ?? null,
                             element_type: action.element_type ?? null,
-                            coordinates: normalizedCoords,
-                            data: {
-                                ...(action.value !== undefined && action.value !== null && { value: action.value_type === "secret" ? "***" : action.value }),
-                                ...(action.mode && { mode: action.mode }),
-                                ...(action.submit && { submit: action.submit }),
-                                ...(action.direction && { direction: action.direction }),
-                                ...(action.amount && { amount: action.amount }),
-                                ...(action.count && action.count > 1 && { count: action.count }),
-                                ...(action.duration_ms && { duration_ms: action.duration_ms }),
-                                ...(action.modifiers?.length && { modifiers: action.modifiers }),
-                                ...(action.key && { key: action.key }),
-                                ...(action.tab_id && { tab_id: action.tab_id }),
-                                ...(action.orientation && { orientation: action.orientation }),
-                                ...(action.panel && { panel: action.panel }),
-                                // The recorded `coordinates` is the drag START; persist the END
-                                // (normalized 0-1000) too so the journey captures the full path.
-                                ...(action.drag && { drag_end: { x: action.drag.endX, y: action.drag.endY } }),
-                                ...(openedNewTab && { opened_new_tab: true }),
-                            },
+                            // Drag's path lives in data.coordinates; the hosted sim leaves the
+                            // top-level coordinates null for a drag.
+                            coordinates: action.drag ? null : normalizedCoords,
+                            data: Object.keys(actionData).length ? actionData : null,
                             order: i,
                         });
                     }
@@ -494,6 +571,15 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                         // Native: drive FrameSourceType.ANDROID/IOS directly; browser falls
                         // back to screen_format server-side.
                         platform,
+                        // Frame continuity: these describe the transition INTO this
+                        // observation, produced by the PREVIOUS step's action. When that
+                        // step was a pure scroll / non-submitting keyboard on a native
+                        // device, the logical screen didn't change — tell the backend to
+                        // reuse the previous frame instead of minting a new one off the
+                        // shifted pixels. Carried from lastStepKind / lastFrameVersionId,
+                        // updated AFTER this call for the next iteration.
+                        ...(isNative && lastFrameVersionId ? { previous_frame_version_id: lastFrameVersionId } : {}),
+                        same_screen_continuation: isNative && (lastStepKind === "scroll" || lastStepKind === "keyboard"),
                     });
                     frameVersionId = matchResult.frame_version_id;
                 }
@@ -501,6 +587,11 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     const msg = err instanceof Error ? err.message : String(err);
                     log(`    Warning: frame matching failed — ${msg}`);
                 }
+                // Carry THIS step's logical-screen classification + matched frame
+                // forward for the NEXT iteration's match-frame call (consumed above as
+                // last*). Classify after the call so ordering is consume-then-update.
+                lastStepKind = classifyStepKind(stepResponse.actions, perActionSuccess);
+                lastFrameVersionId = frameVersionId;
                 // Debug-only: capture post-action screenshot to show result
                 let postActionBase64;
                 if (isDebugEnabled()) {
@@ -520,7 +611,7 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     forwards.push({ type: "LOOP_DETECTED", content: "A repetitive action cycle was detected. Try a different approach." });
                 }
                 // Record interaction (1-indexed step for backend)
-                interactions.push({
+                const interaction = {
                     step: step + 1,
                     assignment_id: assignment.id,
                     ...(screenshotUrl ? { screenshot_url: screenshotUrl } : { screenshot_base64: obsBase64 }),
@@ -544,7 +635,24 @@ async function runSingleSimulation(client, participantId, participantName, opts,
                     // Server reduces this to Interaction.tab when N >= 2; omit on
                     // single-tab steps to keep the payload (and DB column) null.
                     ...(tabsSnapshot.length >= 2 ? { tabs: tabsSnapshot } : {}),
-                });
+                };
+                // Keep the in-memory array for the debug HTML report.
+                interactions.push(interaction);
+                // Stream this interaction live so the backend persists + commits it
+                // immediately and fires INTERACTION_CREATED in realtime. A streaming
+                // failure must never abort the run — log and continue (the run-end
+                // finalize call still records the terminal state).
+                try {
+                    await client.localSimRecordInteraction({
+                        participant_id: session.participant_id,
+                        product_id: session.product_id,
+                        interaction,
+                    });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    log(`  Warning: failed to stream interaction ${step + 1} — ${msg}`);
+                }
                 // Update history for next step
                 history.push({
                     comment: stepResponse.comment,
@@ -635,7 +743,6 @@ async function runSingleSimulation(client, participantId, participantName, opts,
             await client.localSimRecord({
                 participant_id: session.participant_id,
                 product_id: session.product_id,
-                interactions,
                 final_status: finalStatus,
                 assignment_statuses: assignmentStatuses,
             });

package/dist/lib/local-sim/simctl.d.ts

@@ -53,3 +53,18 @@ export declare function isAppInstalled(udid: string, bundleId: string): Promise<
  * terminate+launch a just-installed app without diffing the app list.
  */
 export declare function bundleIdFromApp(appPath: string): Promise<string | null>;
+/**
+ * Read the installed app's marketing version + build number from the booted
+ * simulator. `simctl listapps` emits a (NeXTSTEP) plist of every installed
+ * bundle; we round-trip it through `plutil -convert json` and index by bundle
+ * id. JSON-not-keypath because a bundle id's dots (`com.apple.Preferences`)
+ * collide with plutil's `-extract` keypath separator.
+ *
+ * Best-effort: returns null on any failure (the run never depends on it). Works
+ * for both CLI-installed `.app`s and pre-installed system/app-store bundles —
+ * by call time the bundle id is already resolved.
+ */
+export declare function appBuildFromSimulator(udid: string, bundleId: string): Promise<{
+    version: string | null;
+    build: string | null;
+} | null>;

package/dist/lib/local-sim/simctl.js

@@ -16,7 +16,7 @@
  */
 import { execFile } from "node:child_process";
 import { existsSync } from "node:fs";
-import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { promisify } from "node:util";
@@ -142,3 +142,43 @@ export async function bundleIdFromApp(appPath) {
         return null;
     }
 }
+/**
+ * Read the installed app's marketing version + build number from the booted
+ * simulator. `simctl listapps` emits a (NeXTSTEP) plist of every installed
+ * bundle; we round-trip it through `plutil -convert json` and index by bundle
+ * id. JSON-not-keypath because a bundle id's dots (`com.apple.Preferences`)
+ * collide with plutil's `-extract` keypath separator.
+ *
+ * Best-effort: returns null on any failure (the run never depends on it). Works
+ * for both CLI-installed `.app`s and pre-installed system/app-store bundles —
+ * by call time the bundle id is already resolved.
+ */
+export async function appBuildFromSimulator(udid, bundleId) {
+    const dir = await mkdtemp(join(tmpdir(), "ish-ios-apps-"));
+    const path = join(dir, "apps.plist");
+    try {
+        const { stdout } = await execFileAsync(XCRUN, ["simctl", "listapps", udid], {
+            timeout: 60_000,
+            maxBuffer: 16 * 1024 * 1024,
+        });
+        await writeFile(path, stdout);
+        const { stdout: json } = await execFileAsync(PLUTIL, ["-convert", "json", "-o", "-", path], {
+            timeout: 10_000,
+            maxBuffer: 16 * 1024 * 1024,
+        });
+        const apps = JSON.parse(json);
+        const app = apps[bundleId];
+        if (!app)
+            return null;
+        return {
+            version: app.CFBundleShortVersionString ?? null,
+            build: app.CFBundleVersion ?? null,
+        };
+    }
+    catch {
+        return null;
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true }).catch(() => { });
+    }
+}