@autifyhq/aximo-cli 0.8.0 → 0.10.0

package/dist/mobile-control/appium-controller.d.ts

@@ -12,6 +12,12 @@ export interface AppiumControllerOptions {
      * isn't picked long before it is bound.
      */
     wdaLocalPort?: number | (() => Promise<number>);
+    /**
+     * Local port the driver forwards to the on-device MJPEG server (WDA / the
+     * UiAutomator2 server). When set, the session is created with the MJPEG
+     * server enabled and conservative capture settings applied.
+     */
+    mjpegServerPort?: number | (() => Promise<number>);
     transport?: WebDriverTransport;
 }
 export declare class AppiumController implements MobileController {
@@ -20,12 +26,28 @@ export declare class AppiumController implements MobileController {
     private readonly target;
     private readonly iosSigning;
     private readonly wdaLocalPort;
+    private readonly mjpegServerPort;
     private resolvedWdaPort;
+    private resolvedMjpegPort;
     private sessionReady;
     private scaling;
     constructor(opts: AppiumControllerOptions);
     private capabilities;
     private ensureSession;
+    private static readonly MJPEG_SETTINGS;
+    private applyMjpegSettings;
+    private resetSession;
+    /**
+     * Run a command against a live session, rebuilding the session once if the
+     * cached one has died (WDA killed for CPU, device rebooted, adb dropped).
+     * Without this, one on-device crash permanently wedges every later command —
+     * including the screenshot fallback — until the CLI is restarted.
+     */
+    private withSession;
+    /** Bootstrap (or rebuild) the device session without running a command. */
+    ensureReady(): Promise<void>;
+    /** Local port of the driver-forwarded MJPEG server, once the session is up. */
+    getMjpegPort(): number | undefined;
     private scale;
     private toPoint;
     private pointerAction;
@@ -34,14 +56,20 @@ export declare class AppiumController implements MobileController {
     doubleTap(x: number, y: number): Promise<void>;
     longPress(x: number, y: number, durationMs: number): Promise<void>;
     swipe(x1: number, y1: number, x2: number, y2: number, durationMs: number): Promise<void>;
+    pinchZoom(x: number, y: number, scale: number, durationMs: number): Promise<void>;
+    private pinchZoomOnce;
     private performPointer;
     typeText(text: string): Promise<ActionResult>;
+    private activeElementOrNull;
     clearText(): Promise<ActionResult>;
     pressKey(key: string): Promise<ActionResult>;
+    private pressKeyOnce;
     launchApp(app: string): Promise<ActionResult>;
     findElement(text: string): Promise<ActionResult>;
     private locate;
     hideKeyboard(): Promise<ActionResult>;
+    rotateDevice(orientation: "portrait" | "landscape"): Promise<ActionResult>;
+    setLocation(latitude: number, longitude: number, altitude: number): Promise<ActionResult>;
     getDeviceTime(): Promise<{
         datetime: string;
         timezone: string;

package/dist/mobile-control/appium-controller.js

@@ -12,6 +12,8 @@ const ANDROID_KEY_CODES = {
 };
 const ANDROID_KEYCODE_A = 29;
 const ANDROID_META_CTRL_ON = 4096;
+// Chars that break Android's `mobile: type` (adb shell input text): shell metachars, non-ASCII.
+const ANDROID_INPUT_TEXT_UNSAFE = /[^A-Za-z0-9 @:,./+_=-]/;
 const IOS_BUTTONS = {
     home: "home",
 };
@@ -22,13 +24,22 @@ const W3C_KEYS = {
     escape: "\uE00C",
     space: " ",
 };
+// A dead-but-cached Appium session (WDA killed for CPU, phone rebooted, adb
+// dropped) answers every command with one of these. Distinct from command
+// failures on a live session, which must not trigger a session rebuild.
+const SESSION_DEAD_PATTERN = /invalid session id|terminated or not started|session deleted|session not found|has already finished/i;
+function isSessionDeadError(err) {
+    return err instanceof Error && SESSION_DEAD_PATTERN.test(err.message);
+}
 export class AppiumController {
     session;
     platform;
     target;
     iosSigning;
     wdaLocalPort;
+    mjpegServerPort;
     resolvedWdaPort;
+    resolvedMjpegPort;
     sessionReady = null;
     scaling = null;
     constructor(opts) {
@@ -36,10 +47,14 @@ export class AppiumController {
         this.target = opts.target;
         this.iosSigning = opts.iosSigning;
         this.wdaLocalPort = opts.wdaLocalPort;
+        this.mjpegServerPort = opts.mjpegServerPort;
         const transport = opts.transport ?? fetchTransport(opts.endpoint);
         this.session = new WebDriverSession(transport);
     }
     capabilities() {
+        const mjpeg = this.resolvedMjpegPort
+            ? { "appium:mjpegServerPort": this.resolvedMjpegPort }
+            : {};
         if (this.platform === "android") {
             return {
                 platformName: "Android",
@@ -48,6 +63,7 @@ export class AppiumController {
                 "appium:noReset": true,
                 "appium:autoLaunch": false,
                 "appium:newCommandTimeout": 0,
+                ...mjpeg,
             };
         }
         return {
@@ -57,6 +73,7 @@ export class AppiumController {
             "appium:newCommandTimeout": 0,
             // Unique local->device WDA port so a second connected phone doesn't collide on 8100.
             ...(this.resolvedWdaPort ? { "appium:wdaLocalPort": this.resolvedWdaPort } : {}),
+            ...mjpeg,
             // WDA code-signing caps for physical iPhones (empty unless a team id is set).
             ...(this.iosSigning ? iosSigningCapabilities(this.iosSigning) : {}),
         };
@@ -66,11 +83,14 @@ export class AppiumController {
             this.sessionReady = (async () => {
                 this.resolvedWdaPort =
                     typeof this.wdaLocalPort === "function" ? await this.wdaLocalPort() : this.wdaLocalPort;
+                this.resolvedMjpegPort =
+                    typeof this.mjpegServerPort === "function"
+                        ? await this.mjpegServerPort()
+                        : this.mjpegServerPort;
                 await this.session.createSession(this.capabilities());
+                await this.applyMjpegSettings();
             })().catch(async (err) => {
-                this.sessionReady = null;
-                // A retry must re-pick a free port — the one resolved here may be taken by then.
-                this.resolvedWdaPort = undefined;
+                this.resetSession();
                 // XCUITest reports session-start failures generically ("xcodebuild failed
                 // with code 65"); enrich with the real reason from the WDA .xcresult and
                 // an actionable hint so the user isn't left with an opaque error.
@@ -82,6 +102,61 @@ export class AppiumController {
         }
         return this.sessionReady;
     }
+    // Conservative stream shape until runtime adaptation lands: iOS defaults to
+    // scalingFactor 100 (full-res frames) which can drive WDA CPU high enough for
+    // the OS to kill it, so both platforms are pinned to the same explicit values.
+    static MJPEG_SETTINGS = {
+        mjpegServerFramerate: 10,
+        mjpegScalingFactor: 50,
+        mjpegServerScreenshotQuality: 40,
+    };
+    async applyMjpegSettings() {
+        if (!this.resolvedMjpegPort)
+            return;
+        try {
+            await this.session.updateSettings(AppiumController.MJPEG_SETTINGS);
+        }
+        catch (err) {
+            console.error(`MJPEG capture settings were not applied (streaming continues with driver defaults): ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    resetSession() {
+        this.sessionReady = null;
+        this.scaling = null;
+        // A retry must re-pick free ports — the ones resolved here may be taken by then.
+        this.resolvedWdaPort = undefined;
+        this.resolvedMjpegPort = undefined;
+    }
+    /**
+     * Run a command against a live session, rebuilding the session once if the
+     * cached one has died (WDA killed for CPU, device rebooted, adb dropped).
+     * Without this, one on-device crash permanently wedges every later command —
+     * including the screenshot fallback — until the CLI is restarted.
+     */
+    async withSession(fn) {
+        const ready = this.ensureSession();
+        await ready;
+        try {
+            return await fn();
+        }
+        catch (err) {
+            if (!isSessionDeadError(err))
+                throw err;
+            // Only reset if no concurrent command already rebuilt the session.
+            if (this.sessionReady === ready)
+                this.resetSession();
+            await this.ensureSession();
+            return await fn();
+        }
+    }
+    /** Bootstrap (or rebuild) the device session without running a command. */
+    async ensureReady() {
+        await this.ensureSession();
+    }
+    /** Local port of the driver-forwarded MJPEG server, once the session is up. */
+    getMjpegPort() {
+        return this.resolvedMjpegPort;
+    }
     // Incoming pixel coords scale to logical points: iOS divides by DPR, Android ~= 1.
     async scale() {
         if (this.scaling)
@@ -107,104 +182,176 @@ export class AppiumController {
         ];
     }
     async screenshot() {
-        await this.ensureSession();
-        return this.session.screenshot();
+        return this.withSession(() => this.session.screenshot());
     }
     async tap(x, y) {
-        await this.ensureSession();
-        const [px, py] = await this.toPoint(x, y);
-        await this.performPointer([
-            { type: "pointerMove", duration: 0, x: px, y: py },
-            { type: "pointerDown", button: 0 },
-            { type: "pause", duration: 50 },
-            { type: "pointerUp", button: 0 },
-        ]);
+        await this.withSession(async () => {
+            const [px, py] = await this.toPoint(x, y);
+            await this.performPointer([
+                { type: "pointerMove", duration: 0, x: px, y: py },
+                { type: "pointerDown", button: 0 },
+                { type: "pause", duration: 50 },
+                { type: "pointerUp", button: 0 },
+            ]);
+        });
     }
     async doubleTap(x, y) {
-        await this.ensureSession();
-        const [px, py] = await this.toPoint(x, y);
-        await this.performPointer([
-            { type: "pointerMove", duration: 0, x: px, y: py },
-            { type: "pointerDown", button: 0 },
-            { type: "pause", duration: 50 },
-            { type: "pointerUp", button: 0 },
-            { type: "pause", duration: 50 },
-            { type: "pointerDown", button: 0 },
-            { type: "pause", duration: 50 },
-            { type: "pointerUp", button: 0 },
-        ]);
+        await this.withSession(async () => {
+            const [px, py] = await this.toPoint(x, y);
+            await this.performPointer([
+                { type: "pointerMove", duration: 0, x: px, y: py },
+                { type: "pointerDown", button: 0 },
+                { type: "pause", duration: 50 },
+                { type: "pointerUp", button: 0 },
+                { type: "pause", duration: 50 },
+                { type: "pointerDown", button: 0 },
+                { type: "pause", duration: 50 },
+                { type: "pointerUp", button: 0 },
+            ]);
+        });
     }
     async longPress(x, y, durationMs) {
-        await this.ensureSession();
-        const [px, py] = await this.toPoint(x, y);
-        await this.performPointer([
-            { type: "pointerMove", duration: 0, x: px, y: py },
-            { type: "pointerDown", button: 0 },
-            { type: "pause", duration: Math.max(0, Math.round(durationMs)) },
-            { type: "pointerUp", button: 0 },
-        ]);
+        await this.withSession(async () => {
+            const [px, py] = await this.toPoint(x, y);
+            await this.performPointer([
+                { type: "pointerMove", duration: 0, x: px, y: py },
+                { type: "pointerDown", button: 0 },
+                { type: "pause", duration: Math.max(0, Math.round(durationMs)) },
+                { type: "pointerUp", button: 0 },
+            ]);
+        });
     }
     async swipe(x1, y1, x2, y2, durationMs) {
-        await this.ensureSession();
-        const [px1, py1] = await this.toPoint(x1, y1);
-        const [px2, py2] = await this.toPoint(x2, y2);
-        await this.performPointer([
-            { type: "pointerMove", duration: 0, x: px1, y: py1 },
-            { type: "pointerDown", button: 0 },
-            { type: "pointerMove", duration: Math.max(0, Math.round(durationMs)), x: px2, y: py2 },
-            { type: "pointerUp", button: 0 },
+        await this.withSession(async () => {
+            const [px1, py1] = await this.toPoint(x1, y1);
+            const [px2, py2] = await this.toPoint(x2, y2);
+            await this.performPointer([
+                { type: "pointerMove", duration: 0, x: px1, y: py1 },
+                { type: "pointerDown", button: 0 },
+                { type: "pointerMove", duration: Math.max(0, Math.round(durationMs)), x: px2, y: py2 },
+                { type: "pointerUp", button: 0 },
+            ]);
+        });
+    }
+    // Two-finger vertical pinch: scale < 1 pinches in (zoom out), > 1 spreads (zoom in).
+    async pinchZoom(x, y, scale, durationMs) {
+        await this.withSession(() => this.pinchZoomOnce(x, y, scale, durationMs));
+    }
+    async pinchZoomOnce(x, y, scale, durationMs) {
+        const [cx, cy] = await this.toPoint(x, y);
+        const startOffset = 100;
+        const endOffset = startOffset * scale;
+        const duration = Math.max(0, Math.round(durationMs));
+        const finger = (id, startY, endY) => ({
+            type: "pointer",
+            id,
+            parameters: { pointerType: "touch" },
+            actions: [
+                { type: "pointerMove", duration: 0, x: cx, y: Math.round(startY) },
+                { type: "pointerDown", button: 0 },
+                { type: "pause", duration: 50 },
+                { type: "pointerMove", duration, x: cx, y: Math.round(endY) },
+                { type: "pointerUp", button: 0 },
+            ],
+        });
+        await this.session.performActions([
+            finger("finger1", cy - startOffset / 2, cy - endOffset / 2),
+            finger("finger2", cy + startOffset / 2, cy + endOffset / 2),
         ]);
+        await this.session.releaseActions();
     }
     async performPointer(actions) {
         await this.session.performActions(this.pointerAction(actions));
         await this.session.releaseActions();
     }
     async typeText(text) {
-        await this.ensureSession();
         try {
-            if (this.platform === "android") {
-                await this.session.execute("mobile: type", [{ text }]);
+            return await this.withSession(async () => {
+                if (this.platform === "android") {
+                    if (ANDROID_INPUT_TEXT_UNSAFE.test(text)) {
+                        const active = await this.activeElementOrNull();
+                        if (!active) {
+                            return {
+                                ok: false,
+                                error: "Cannot type special or non-ASCII characters: no input field is focused. Tap the field first.",
+                            };
+                        }
+                        await this.session.elementSendKeys(active, text);
+                        return { ok: true };
+                    }
+                    await this.session.execute("mobile: type", [{ text }]);
+                    return { ok: true };
+                }
+                // XCUITest: per-char W3C key actions; no focused element required.
+                const actions = [...text].flatMap((char) => [
+                    { type: "keyDown", value: char },
+                    { type: "keyUp", value: char },
+                ]);
+                await this.session.performActions([{ type: "key", id: "keyboard", actions }]);
+                await this.session.releaseActions();
                 return { ok: true };
-            }
-            await this.session.setValueOnActiveElement(text);
-            return { ok: true };
+            });
         }
         catch (error) {
             return fail(error);
         }
     }
+    // Some drivers report "no focus" by rejecting; treat that as null.
+    async activeElementOrNull() {
+        try {
+            return await this.session.getActiveElement();
+        }
+        catch {
+            return null;
+        }
+    }
     async clearText() {
-        await this.ensureSession();
         try {
-            const active = await this.session.getActiveElement();
-            if (active) {
-                await this.session.elementClear(active);
-                return { ok: true };
-            }
+            return await this.withSession(async () => {
+                try {
+                    const active = await this.session.getActiveElement();
+                    if (active) {
+                        await this.session.elementClear(active);
+                        return { ok: true };
+                    }
+                }
+                catch (error) {
+                    // Re-throw a dead session so withSession can rebuild it; "no focused
+                    // element" style rejections fall through to the fallback path.
+                    if (isSessionDeadError(error))
+                        throw error;
+                }
+                if (this.platform === "android") {
+                    await this.session.execute("mobile: pressKey", [
+                        { keycode: ANDROID_KEYCODE_A, metastate: ANDROID_META_CTRL_ON },
+                    ]);
+                    await this.session.execute("mobile: pressKey", [
+                        { keycode: ANDROID_KEY_CODES["delete"] },
+                    ]);
+                    return { ok: true };
+                }
+                return {
+                    ok: false,
+                    error: "Could not clear the field: no focused input element was found. " +
+                        "Tap the field to focus it before calling clear_text.",
+                };
+            });
         }
-        catch { }
-        if (this.platform === "android") {
-            try {
-                await this.session.execute("mobile: pressKey", [
-                    { keycode: ANDROID_KEYCODE_A, metastate: ANDROID_META_CTRL_ON },
-                ]);
-                await this.session.execute("mobile: pressKey", [{ keycode: ANDROID_KEY_CODES["delete"] }]);
-                return { ok: true };
-            }
-            catch (error) {
-                return fail(error);
-            }
+        catch (error) {
+            return fail(error);
         }
-        return {
-            ok: false,
-            error: "Could not clear the field: no focused input element was found. " +
-                "Tap the field to focus it before calling clear_text.",
-        };
     }
     async pressKey(key) {
-        await this.ensureSession();
         const normalized = key.toLowerCase().trim();
         try {
+            return await this.withSession(() => this.pressKeyOnce(key, normalized));
+        }
+        catch (error) {
+            return fail(error);
+        }
+    }
+    async pressKeyOnce(key, normalized) {
+        {
             if (this.platform === "android") {
                 const code = ANDROID_KEY_CODES[normalized];
                 if (code === undefined)
@@ -240,18 +387,16 @@ export class AppiumController {
             }
             return { ok: false, error: `Unsupported key: ${key}` };
         }
-        catch (error) {
-            return fail(error);
-        }
     }
     async launchApp(app) {
         if (!app)
             return { ok: false, error: "launch_app requires an app id" };
-        await this.ensureSession();
         try {
-            const param = this.platform === "android" ? { appId: app } : { bundleId: app };
-            await this.session.execute("mobile: activateApp", [param]);
-            return { ok: true };
+            return await this.withSession(async () => {
+                const param = this.platform === "android" ? { appId: app } : { bundleId: app };
+                await this.session.execute("mobile: activateApp", [param]);
+                return { ok: true };
+            });
         }
         catch (error) {
             return fail(error);
@@ -260,13 +405,14 @@ export class AppiumController {
     async findElement(text) {
         if (!text)
             return { ok: false, error: "find_element requires text" };
-        await this.ensureSession();
         try {
-            const ref = await this.locate(text);
-            if (!ref)
-                return { ok: false, error: `Element with text "${text}" not found` };
-            await this.session.elementClick(ref);
-            return { ok: true };
+            return await this.withSession(async () => {
+                const ref = await this.locate(text);
+                if (!ref)
+                    return { ok: false, error: `Element with text "${text}" not found` };
+                await this.session.elementClick(ref);
+                return { ok: true };
+            });
         }
         catch (error) {
             return fail(error);
@@ -277,7 +423,8 @@ export class AppiumController {
         for (const [using, value] of strategies) {
             try {
                 const ref = await this.session.findElement(using, value);
-                if (ref)
+                // Skip hidden matches so the next strategy can find a visible one.
+                if (ref && (await this.session.elementDisplayed(ref)))
                     return ref;
             }
             catch { }
@@ -285,19 +432,51 @@ export class AppiumController {
         return null;
     }
     async hideKeyboard() {
-        await this.ensureSession();
         try {
-            await this.session.execute("mobile: hideKeyboard", []);
-            return { ok: true };
+            await this.withSession(() => this.session.execute("mobile: hideKeyboard", []));
         }
-        catch {
-            return { ok: true };
+        catch { }
+        return { ok: true };
+    }
+    async rotateDevice(orientation) {
+        try {
+            return await this.withSession(async () => {
+                const wantLandscape = orientation === "landscape";
+                await this.session.setOrientation(wantLandscape ? "LANDSCAPE" : "PORTRAIT");
+                // Rotation changes the window rect, so the cached pixel->point scale is stale.
+                this.scaling = null;
+                // Poll up to 2s until the OS reports the new dimensions; rotation speed varies.
+                const deadline = Date.now() + 2000;
+                while (Date.now() < deadline) {
+                    try {
+                        const rect = await this.session.windowRect();
+                        if (rect.width > rect.height === wantLandscape)
+                            break;
+                    }
+                    catch { }
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                }
+                return { ok: true };
+            });
+        }
+        catch (error) {
+            return fail(error);
+        }
+    }
+    async setLocation(latitude, longitude, altitude) {
+        try {
+            return await this.withSession(async () => {
+                await this.session.setGeoLocation({ latitude, longitude, altitude });
+                return { ok: true };
+            });
+        }
+        catch (error) {
+            return fail(error);
         }
     }
     async getDeviceTime() {
-        await this.ensureSession();
         try {
-            const raw = await this.session.execute("mobile: getDeviceTime", []);
+            const raw = await this.withSession(() => this.session.execute("mobile: getDeviceTime", []));
             if (typeof raw !== "string" || raw.trim().length === 0)
                 return null;
             const parsed = new Date(raw);