@xenon-device-management/xenon 2.13.2 → 2.15.0

package/lib/test/unit/omni-virtual-element-store.spec.js

@@ -0,0 +1,301 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const plugin_1 = require("../../src/plugin");
+const NodeDevices_1 = __importDefault(require("../../src/device-managers/NodeDevices"));
+const DeviceService = __importStar(require("../../src/data-service/device-service"));
+const ActiveLeases = __importStar(require("../../src/services/lease/activeLeases"));
+const device_utils_1 = require("../../src/device-utils");
+const CommandInterceptor_1 = require("../../src/interceptors/CommandInterceptor");
+const HealingOrchestrator_1 = require("../../src/services/healing/HealingOrchestrator");
+const types_1 = require("../../src/services/healing/types");
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const AutowaitService_1 = require("../../src/services/autowait/AutowaitService");
+const SessionLifecycleService_1 = require("../../src/services/SessionLifecycleService");
+const SessionMetricsService_1 = require("../../src/services/metrics/SessionMetricsService");
+const OrphanSweeper_1 = require("../../src/services/OrphanSweeper");
+const PhoneNetworkRestore_1 = require("../../src/services/network/PhoneNetworkRestore");
+const SelfHealingSwitch_1 = require("../../src/services/settings/SelfHealingSwitch");
+const AIService_1 = require("../../src/services/AIService");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * An element Xenon finds in a screenshot (a `-custom:ai-*` find, or an OCR or
+ * Visual AI heal) is a box on the screen it remembers by id. Through 2.14 it
+ * remembered every one, from every session, until the server restarted:
+ * device control's locator test and an icon `smartTap` added one per call
+ * that nothing ever read, and any session could act on another's. Now an
+ * element belongs to the session that found it and goes when that session
+ * ends, and the store has a size limit.
+ *
+ * OCR, the AI provider and the drivers are stubbed: no Tesseract, no AI call,
+ * no phone.
+ */
+const W3C = 'element-6066-11e4-a52e-4f735466cecf';
+const WORDS = [
+    { text: 'Sign', confidence: 90, bbox: { x0: 100, y0: 200, x1: 150, y1: 224 } },
+    { text: 'in', confidence: 90, bbox: { x0: 158, y0: 200, x1: 180, y1: 224 } },
+];
+const box = (id) => ({
+    id,
+    rect: { x: 0, y: 0, width: 10, height: 10 },
+    confidence: 0.9,
+});
+describe('Elements found in a screenshot belong to their session', () => {
+    (0, scratch_database_1.useScratchDatabase)();
+    let restoreContainer;
+    let omni;
+    const driverFor = (sessionId) => ({
+        sessionId,
+        caps: { platformName: 'Android' },
+        getScreenshot: sinon_1.default.stub().resolves('aGVsbG8='),
+        performActions: sinon_1.default.stub().resolves(),
+        releaseActions: sinon_1.default.stub().resolves(),
+    });
+    beforeEach(() => {
+        restoreContainer = (0, container_registration_1.saveRegistrations)(OmniVisionService_1.OmniVisionService, HealingOrchestrator_1.HealingOrchestrator, SelfHealingSwitch_1.SelfHealingSwitch);
+        typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+        typedi_1.Container.set(SelfHealingSwitch_1.SelfHealingSwitch, new SelfHealingSwitch_1.SelfHealingSwitch());
+        omni = typedi_1.Container.get(OmniVisionService_1.OmniVisionService);
+        sinon_1.default.stub(omni, 'performOcr').resolves({ text: '', words: WORDS });
+        sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'afterSessionCommand').resolves();
+    });
+    afterEach(() => {
+        restoreContainer();
+        sinon_1.default.restore();
+    });
+    const run = (driver, command, args, next = () => null) => typedi_1.Container.get(CommandInterceptor_1.CommandInterceptor).handle(() => __awaiter(void 0, void 0, void 0, function* () { return next(); }), driver, command, [...args, driver.sessionId], Object.assign({}, IPluginArgs_1.DefaultPluginArgs), false);
+    const failure = (promise) => __awaiter(void 0, void 0, void 0, function* () {
+        try {
+            yield promise;
+        }
+        catch (err) {
+            return err;
+        }
+        throw new Error('expected the command to fail');
+    });
+    const held = () => omni.elements.size;
+    it("answers another session's command on it with no such element", () => __awaiter(void 0, void 0, void 0, function* () {
+        const a = driverFor('sess-a');
+        const b = driverFor('sess-b');
+        const found = yield run(a, 'findElement', ['-custom:ai-text', 'Sign in']);
+        const err = yield failure(run(b, 'click', [found[W3C]]));
+        (0, chai_1.expect)(err.error).to.equal('no such element');
+        (0, chai_1.expect)(b.performActions.called, "tapped on the other session's phone").to.equal(false);
+        yield run(a, 'click', [found[W3C]]);
+        (0, chai_1.expect)(a.performActions.calledOnce).to.equal(true);
+    }));
+    it('keeps a healed element for the session it healed in', () => __awaiter(void 0, void 0, void 0, function* () {
+        const attemptHealing = sinon_1.default.stub().resolves({
+            id: 'healed_ocr_1_abc',
+            tier: types_1.HealingTier.TIER_3_LOCAL_OCR,
+            confidence: 0.9,
+            originalSelector: "//*[@text='Sign in']",
+            originalStrategy: 'xpath',
+            recommendedSelector: 'ocr:text="Sign in"',
+            recommendedStrategy: 'xenon:visual',
+            text: 'Sign in',
+            rect: { x: 100, y: 200, width: 80, height: 24 },
+        });
+        typedi_1.Container.set(HealingOrchestrator_1.HealingOrchestrator, { attemptHealing });
+        const a = driverFor('sess-a');
+        yield run(a, 'findElement', ['xpath', "//*[@text='Sign in']"], () => {
+            throw Object.assign(new Error('NoSuchElement'), { name: 'NoSuchElementError' });
+        });
+        (0, chai_1.expect)(yield run(a, 'getText', ['healed_ocr_1_abc'])).to.equal('Sign in');
+        const err = yield failure(run(driverFor('sess-b'), 'getText', ['healed_ocr_1_abc']));
+        (0, chai_1.expect)(err.error).to.equal('no such element');
+    }));
+    it("keeps nothing for device control's locator test", () => __awaiter(void 0, void 0, void 0, function* () {
+        // As POST /control/<udid>/test-locator calls it: the ids are only shown.
+        const manual = { sessionId: 'manual_R5CT', getScreenshot: sinon_1.default.stub().resolves('aGVsbG8=') };
+        const found = yield omni.findByText(manual, 'Sign in', { throwOnError: true });
+        (0, chai_1.expect)(found).to.have.length(1);
+        (0, chai_1.expect)(held()).to.equal(0);
+    }));
+    it('keeps nothing for an icon smartTap, which taps at once', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 50, y: 50 });
+        const result = yield omni.omniClickByIcon(driverFor('sess-a'), { icon: 'the cart' });
+        (0, chai_1.expect)(result.clicked).to.equal(true);
+        (0, chai_1.expect)(held()).to.equal(0);
+    }));
+    it('keeps the newest 1,000 for a session', () => {
+        for (let i = 0; i < 1001; i++)
+            omni.remember('sess-a', box(`omni_ocr_${i}`));
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_0', 'sess-a')).to.equal(undefined);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_1', 'sess-a')).to.not.equal(undefined);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_1000', 'sess-a')).to.not.equal(undefined);
+    });
+    it('keeps the newest 10,000 in all', () => {
+        for (let s = 0; s < 11; s++) {
+            for (let i = 0; i < 1000; i++)
+                omni.remember(`sess-${s}`, box(`omni_ocr_${s}_${i}`));
+        }
+        (0, chai_1.expect)(held()).to.equal(10000);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_0_999', 'sess-0')).to.equal(undefined);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_1_0', 'sess-1')).to.not.equal(undefined);
+    });
+    it('gives icon matches ids of their own, as text matches have', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(Date, 'now').returns(1700000000000);
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 50, y: 50 });
+        const a = driverFor('sess-a');
+        const first = yield omni.findByIcon(a, 'the cart');
+        const second = yield omni.findByIcon(a, 'the cart');
+        (0, chai_1.expect)(first === null || first === void 0 ? void 0 : first.id).to.not.equal(second === null || second === void 0 ? void 0 : second.id);
+    }));
+});
+/**
+ * Everything a session leaves in this server's memory for its commands goes
+ * when the session ends, however it ends: its virtual elements, and its
+ * autowait settings (`xenon: setAutowaitProperties`). Those were meant to be
+ * cleared by the interceptor on `deleteSession`, but XenonPlugin answers
+ * `deleteSession` itself, so Appium never hands that command to the
+ * interceptor: they were kept for good.
+ */
+describe("A session's virtual elements and autowait settings go when it ends", function () {
+    this.timeout(30000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let restoreRegs;
+    let omni;
+    const seed = (sessionId) => {
+        omni.remember(sessionId, box(`omni_ocr_${sessionId}`));
+        typedi_1.Container.get(AutowaitService_1.AutowaitService).setProps(sessionId, { timeoutMs: 1234 });
+    };
+    const expectForgotten = (sessionId) => {
+        (0, chai_1.expect)(omni.getVirtualElement(`omni_ocr_${sessionId}`, sessionId), 'element').to.equal(undefined);
+        (0, chai_1.expect)(typedi_1.Container.get(AutowaitService_1.AutowaitService).getProps(sessionId, Object.assign({}, IPluginArgs_1.DefaultPluginArgs)).timeoutMs, 'autowait timeout').to.equal(IPluginArgs_1.DefaultPluginArgs.autowait.timeoutMs);
+    };
+    beforeEach(() => {
+        restoreRegs = (0, container_registration_1.saveRegistrations)(OmniVisionService_1.OmniVisionService, AutowaitService_1.AutowaitService, PhoneNetworkRestore_1.PhoneNetworkRestore, SessionMetricsService_1.SessionMetricsService);
+        typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+        typedi_1.Container.set(AutowaitService_1.AutowaitService, new AutowaitService_1.AutowaitService());
+        omni = typedi_1.Container.get(OmniVisionService_1.OmniVisionService);
+        typedi_1.Container.set(PhoneNetworkRestore_1.PhoneNetworkRestore, {
+            restoreSession: sinon_1.default.stub().resolves(),
+            restoreAll: sinon_1.default.stub().resolves(),
+        });
+        typedi_1.Container.set(SessionMetricsService_1.SessionMetricsService, { stop: sinon_1.default.stub().resolves() });
+        sinon_1.default.stub(DeviceService, 'releaseSessionDevices').resolves();
+        sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'onSessionStopped').resolves();
+    });
+    afterEach(() => {
+        sinon_1.default.restore();
+        restoreRegs();
+    });
+    it('when the test deletes the session', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-deleted');
+        yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).deleteSession(() => __awaiter(this, void 0, void 0, function* () { return null; }), 's-deleted');
+        expectForgotten('s-deleted');
+    }));
+    it('when Appium ends the session itself (new-command timeout)', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-timeout');
+        const plugin = { pluginArgs: {}, xenonLog: { withSession: () => ({ info() { } }) } };
+        yield plugin_1.XenonPlugin.prototype.onUnexpectedShutdown.call(plugin, { sessionId: 's-timeout', caps: { udid: 'R5CT' } }, new Error('New command timeout'));
+        expectForgotten('s-timeout');
+    }));
+    it('when Appium ends the session on a node', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-node');
+        sinon_1.default.stub(NodeDevices_1.default.prototype, 'unblockDevice').resolves(undefined);
+        const plugin = { pluginArgs: { hub: 'http://hub:4723' } };
+        yield plugin_1.XenonPlugin.prototype.onUnexpectedShutdown.call(plugin, { sessionId: 's-node', caps: {} }, null);
+        expectForgotten('s-node');
+    }));
+    it("when Xenon's idle check releases the phone", () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-idle');
+        const idle = {
+            udid: 'R5CT',
+            host: 'http://127.0.0.1:4723',
+            busy: true,
+            userBlocked: false,
+            session_id: 's-idle',
+            claimSessionId: 's-idle',
+            lastCmdExecutedAt: Date.now() - 10 * 60 * 1000,
+        };
+        sinon_1.default.stub(DeviceService, 'getAllDevices').resolves([idle]);
+        sinon_1.default.stub(ActiveLeases, 'activeLeasesByDevice').resolves(new Map());
+        sinon_1.default.stub(DeviceService, 'releaseSessionDevice').resolves(true);
+        yield (0, device_utils_1.releaseBlockedDevices)(60);
+        expectForgotten('s-idle');
+    }));
+    it('when a graceful shutdown drains the session', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-down');
+        yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).stopSessionForShutdown('s-down', 'shutdown');
+        expectForgotten('s-down');
+    }));
+    it("when the session's heartbeat goes stale", () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-stale');
+        yield scratch.db.session.deleteMany({});
+        yield scratch.db.session.create({
+            data: {
+                id: 's-stale',
+                status: 'running',
+                desired_capabilities: '{}',
+                session_capabilities: '{}',
+                node_id: 'n',
+                has_live_video: false,
+                device_udid: 'R5CT',
+                device_platform: 'android',
+                device_version: '14',
+                last_heartbeat_at: new Date(Date.now() - 60 * 60 * 1000),
+            },
+        });
+        yield new OrphanSweeper_1.OrphanSweeper().sweep({ heartbeatIntervalMs: 1000 });
+        expectForgotten('s-stale');
+    }));
+    it("leaves other sessions' elements alone", () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-ends');
+        seed('s-stays');
+        yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).deleteSession(() => __awaiter(this, void 0, void 0, function* () { return null; }), 's-ends');
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_s-stays', 's-stays')).to.not.equal(undefined);
+    }));
+});

package/lib/test/unit/omni-vision-analyze-screen.spec.js

@@ -0,0 +1,72 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const AIService_1 = require("../../src/services/AIService");
+/**
+ * `xenon: analyzeScreen` (and device control's Omni-Scan) sent the screenshot's
+ * base64 where a file path was expected. The AI step found no such file and
+ * asked the provider without an image, under a "why did this test fail"
+ * prompt, so `ai_insights` was generic text about no screen at all.
+ *
+ * Now the screenshot goes to the provider with a prompt about the screen, and
+ * when there is no AI answer `ai_insights_error` says why.
+ */
+describe('xenon: analyzeScreen', () => {
+    const SCREEN = 'c2NyZWVuc2hvdA==';
+    let service;
+    const driver = { sessionId: 'session-1', getScreenshot: () => __awaiter(void 0, void 0, void 0, function* () { return SCREEN; }) };
+    beforeEach(() => {
+        service = new OmniVisionService_1.OmniVisionService();
+        sinon_1.default.stub(service, 'performOcr').resolves({
+            text: 'Sign in',
+            words: [{ text: 'Sign', confidence: 95, bbox: { x0: 10, y0: 10, x1: 50, y1: 30 } }],
+        });
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'initializeProvider');
+    });
+    afterEach(() => sinon_1.default.restore());
+    it('sends the screenshot to the AI provider and asks about the screen', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value({ analyze: () => __awaiter(void 0, void 0, void 0, function* () { return ''; }) });
+        const call = sinon_1.default
+            .stub(AIService_1.AI_SERVICE, 'callProvider')
+            .resolves('A sign-in screen with an email field.');
+        const analysis = yield service.analyzeScreen(driver);
+        (0, chai_1.expect)(call.calledOnce).to.equal(true);
+        const [prompt, image] = call.firstCall.args;
+        (0, chai_1.expect)(image).to.equal(SCREEN);
+        (0, chai_1.expect)(prompt).to.match(/screen/i);
+        (0, chai_1.expect)(prompt).to.not.match(/root cause|failed/i);
+        (0, chai_1.expect)(analysis.ai_insights).to.equal('A sign-in screen with an email field.');
+        (0, chai_1.expect)(analysis).to.not.have.property('ai_insights_error');
+        (0, chai_1.expect)(analysis.ocr.text).to.equal('Sign in');
+    }));
+    it('says why there are no AI insights when no provider is configured', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value(null);
+        const analysis = yield service.analyzeScreen(driver);
+        (0, chai_1.expect)(analysis.ai_insights).to.equal(null);
+        (0, chai_1.expect)(analysis.ai_insights_error).to.match(/No AI provider is configured/);
+        (0, chai_1.expect)(analysis.ocr.text).to.equal('Sign in');
+    }));
+    it('says why there are no AI insights when the provider fails', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value({ analyze: () => __awaiter(void 0, void 0, void 0, function* () { return ''; }) });
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'callProvider').rejects(new Error('429 quota exceeded'));
+        const analysis = yield service.analyzeScreen(driver);
+        (0, chai_1.expect)(analysis.ai_insights).to.equal(null);
+        (0, chai_1.expect)(analysis.ai_insights_error).to.match(/429 quota exceeded/);
+        (0, chai_1.expect)(analysis.ocr.text).to.equal('Sign in');
+    }));
+});

package/lib/test/unit/omni-vision-ios-points.spec.js

@@ -0,0 +1,159 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const sharp_1 = __importDefault(require("sharp"));
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const AIService_1 = require("../../src/services/AIService");
+const HealingOrchestrator_1 = require("../../src/services/healing/HealingOrchestrator");
+const types_1 = require("../../src/services/healing/types");
+const screenScale_1 = require("../../src/services/omni-vision/screenScale");
+/**
+ * OCR and the AI find a target in screenshot pixels. An iPhone's screenshot is
+ * 2x or 3x its screen in points, and XCUITest's W3C actions and element rects
+ * are in points, so a tap at the pixel position landed 2-3 times too far right
+ * and down. Nothing converted: not smartTap/visualTap, not the virtual
+ * elements the -custom:ai-* locators return, not the healing tiers' tap.
+ *
+ * Positions found in a screenshot are now converted to the driver's
+ * coordinates on iOS (screenshot size over window size). Android's W3C
+ * coordinates are the screenshot's pixels, so nothing changes there.
+ */
+describe('Omni-Vision taps in points on iOS', () => {
+    // A 3x screen: 30 x 60 pixels for a 10 x 20 point window.
+    let png3x;
+    before(() => __awaiter(void 0, void 0, void 0, function* () {
+        const image = yield (0, sharp_1.default)({
+            create: { width: 30, height: 60, channels: 3, background: { r: 255, g: 255, b: 255 } },
+        })
+            .png()
+            .toBuffer();
+        png3x = image.toString('base64');
+    }));
+    // "OK" at pixels 12..18 x 30..36: its middle is (15, 33) px, (5, 11) pt.
+    const OK_WORD = { text: 'OK', confidence: 90, bbox: { x0: 12, y0: 30, x1: 18, y1: 36 } };
+    let service;
+    let ios;
+    let android;
+    function driverFor(platformName) {
+        return {
+            sessionId: `session-${platformName}`,
+            caps: { platformName },
+            getScreenshot: sinon_1.default.stub().resolves(png3x),
+            getWindowRect: sinon_1.default.stub().resolves({ x: 0, y: 0, width: 10, height: 20 }),
+            performActions: sinon_1.default.stub().resolves(),
+            releaseActions: sinon_1.default.stub().resolves(),
+        };
+    }
+    beforeEach(() => {
+        service = new OmniVisionService_1.OmniVisionService();
+        sinon_1.default.stub(service, 'performOcr').resolves({ text: 'OK', words: [OK_WORD] });
+        ios = driverFor('iOS');
+        android = driverFor('Android');
+    });
+    afterEach(() => sinon_1.default.restore());
+    const tappedAt = (driver) => {
+        const move = driver.performActions.firstCall.args[0][0].actions[0];
+        return { x: move.x, y: move.y };
+    };
+    it('smartTap by text taps in points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        const result = yield service.omniClickByText(ios, { text: 'OK' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(tappedAt(ios)).to.deep.equal({ x: 5, y: 11 });
+        (0, chai_1.expect)(result.target).to.deep.include({
+            x: 5,
+            y: 11,
+            rect: { x: 4, y: 10, width: 2, height: 2 },
+        });
+    }));
+    it('smartTap by text taps in pixels on Android, without asking for the window', () => __awaiter(void 0, void 0, void 0, function* () {
+        const result = yield service.omniClickByText(android, { text: 'OK' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(tappedAt(android)).to.deep.equal({ x: 15, y: 33 });
+        (0, chai_1.expect)(android.getWindowRect.called).to.equal(false);
+    }));
+    it('visualTap taps in points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 15, y: 33 });
+        const result = yield service.omniClickByIcon(ios, { icon: 'the gear icon' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(tappedAt(ios)).to.deep.equal({ x: 5, y: 11 });
+    }));
+    it('gives the -custom:ai-text locator an element in points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        const [found] = yield service.findByText(ios, 'OK');
+        (0, chai_1.expect)(found.rect).to.deep.equal({ x: 4, y: 10, width: 2, height: 2 });
+    }));
+    it("doesn't tap on iOS when the window size can't be read", () => __awaiter(void 0, void 0, void 0, function* () {
+        ios.getWindowRect.rejects(new Error('WDA is not running'));
+        const err = yield service.omniClickByText(ios, { text: 'OK' }).then(() => null, (e) => e);
+        (0, chai_1.expect)(err === null || err === void 0 ? void 0 : err.message).to.match(/WDA is not running/);
+        (0, chai_1.expect)(ios.performActions.called).to.equal(false);
+    }));
+    it('reads the window from getWindowSize when there is no getWindowRect', () => __awaiter(void 0, void 0, void 0, function* () {
+        const driver = {
+            caps: { platformName: 'iOS' },
+            getWindowSize: () => __awaiter(void 0, void 0, void 0, function* () { return ({ width: 15, height: 30 }); }),
+        };
+        (0, chai_1.expect)(yield (0, screenScale_1.screenScaleOf)(driver, png3x)).to.deep.equal({ x: 2, y: 2 });
+    }));
+    it("keeps pixels for a driver that isn't iOS, or can't be placed", () => __awaiter(void 0, void 0, void 0, function* () {
+        (0, chai_1.expect)(yield (0, screenScale_1.screenScaleOf)({}, 'not an image')).to.deep.equal({ x: 1, y: 1 });
+        (0, chai_1.expect)(yield (0, screenScale_1.screenScaleOf)({ caps: { platformName: 'Android' } }, 'x')).to.deep.equal({
+            x: 1,
+            y: 1,
+        });
+    }));
+    it('converts the OCR and Visual AI healing tiers’ rect to points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        const orchestrator = new HealingOrchestrator_1.HealingOrchestrator({
+            getSignature: () => __awaiter(void 0, void 0, void 0, function* () { return null; }),
+            saveSignature: () => __awaiter(void 0, void 0, void 0, function* () { return undefined; }),
+        });
+        const ocr = orchestrator.providers.find((p) => p.tier === types_1.HealingTier.TIER_3_LOCAL_OCR);
+        sinon_1.default.stub(ocr, 'heal').resolves({
+            id: 'healed_ocr_1',
+            tier: types_1.HealingTier.TIER_3_LOCAL_OCR,
+            confidence: 0.9,
+            originalSelector: '//*[@label="OK"]',
+            recommendedSelector: 'ocr:text="OK"',
+            rect: { x: 12, y: 30, width: 6, height: 6 },
+        });
+        ios.getPageSource = sinon_1.default.stub().resolves('<AppiumAUT/>');
+        const healed = yield orchestrator.attemptHealing('s1', ios, 'xpath', '//*[@label="OK"]', [3]);
+        (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.rect).to.deep.equal({ x: 4, y: 10, width: 2, height: 2 });
+    }));
+    it("drops a healed virtual element whose position on an iPhone can't be worked out", () => __awaiter(void 0, void 0, void 0, function* () {
+        const orchestrator = new HealingOrchestrator_1.HealingOrchestrator({
+            getSignature: () => __awaiter(void 0, void 0, void 0, function* () { return null; }),
+            saveSignature: () => __awaiter(void 0, void 0, void 0, function* () { return undefined; }),
+        });
+        const ocr = orchestrator.providers.find((p) => p.tier === types_1.HealingTier.TIER_3_LOCAL_OCR);
+        const found = {
+            tier: types_1.HealingTier.TIER_3_LOCAL_OCR,
+            confidence: 0.9,
+            originalSelector: '//*[@label="OK"]',
+            recommendedSelector: 'ocr:text="OK"',
+            rect: { x: 12, y: 30, width: 6, height: 6 },
+        };
+        const heal = sinon_1.default.stub(ocr, 'heal').resolves(Object.assign(Object.assign({}, found), { id: 'healed_ocr_1' }));
+        ios.getPageSource = sinon_1.default.stub().resolves('<AppiumAUT/>');
+        ios.getWindowRect.rejects(new Error('WDA is not running'));
+        (0, chai_1.expect)(yield orchestrator.attemptHealing('s1', ios, 'xpath', '//x', [3])).to.equal(null);
+        // A real element the tier resolved is kept, without the rect.
+        heal.resolves(Object.assign(Object.assign({}, found), { id: 'real-element-1' }));
+        const healed = yield orchestrator.attemptHealing('s1', ios, 'xpath', '//x', [3]);
+        (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.id).to.equal('real-element-1');
+        (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.rect).to.equal(undefined);
+    }));
+});