@xenon-device-management/xenon 2.13.2 → 2.14.0

package/lib/test/unit/max-sessions.spec.js

@@ -0,0 +1,240 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = __importDefault(require("chai"));
+const sinon_1 = __importDefault(require("sinon"));
+const DeviceUtils = __importStar(require("../../src/device-utils"));
+const db_1 = require("../../src/data-service/db");
+const device_service_1 = require("../../src/data-service/device-service");
+const deviceClaims_1 = require("../../src/data-service/deviceClaims");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+const test_container_1 = require("../helpers/test-container");
+const loki_stores_1 = require("../helpers/loki-stores");
+const scratch_database_1 = require("../helpers/scratch-database");
+const expect = chai_1.default.expect;
+/**
+ * `maxSessions` holds new sessions back once that many Appium sessions run.
+ *
+ * It compared the number of busy phones with `===`, so once the count was past
+ * the limit nothing was held back any more, and it counted every busy phone: a
+ * live preview, a recording, or an SDK lease nobody had started a session on
+ * used up a slot that no Appium session was using. What it counts now is the
+ * phones running a session, or being given one.
+ */
+describe('maxSessions', () => {
+    describe('which phones count', () => {
+        const base = { busy: true };
+        it('counts a session being created (a claim with no session id yet)', () => {
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { claimedAt: Date.now() }))).to.equal(true);
+        });
+        it('counts a session that is running', () => {
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { claimedAt: 1, claimSessionId: 'sess-1', session_id: 'sess-1' }))).to.equal(true);
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { session_id: 'sess-1' }))).to.equal(true);
+        });
+        it('does not count a live preview or a recording', () => {
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { session_id: 'manual_u1_phone-1' }))).to.equal(false);
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { session_id: 'manual_phone-1' }))).to.equal(false);
+        });
+        it('does not count a phone an SDK lease holds that has no session on it', () => {
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign({}, base))).to.equal(false);
+        });
+        it('counts a session started on a leased phone', () => {
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { claimSessionId: 'sess-2', claimedAt: 1 }))).to.equal(true);
+        });
+        it('does not count a phone that is not busy', () => {
+            expect((0, deviceClaims_1.countsTowardMaxSessions)({ busy: false })).to.equal(false);
+            expect((0, deviceClaims_1.countsTowardMaxSessions)({ busy: false, session_id: 'sess-1' })).to.equal(false);
+        });
+        it("counts a node's phone the node reports busy, unless the node says it is a preview", () => {
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { nodeBusy: true }))).to.equal(true);
+            expect((0, deviceClaims_1.countsTowardMaxSessions)(Object.assign(Object.assign({}, base), { nodeBusy: true, nodeHold: 'manual_u1_phone-1' }))).to.equal(false);
+        });
+    });
+    describe('the limit itself', () => {
+        it('is the setting when it is a positive number', () => {
+            expect((0, deviceClaims_1.sessionCap)(8)).to.equal(8);
+            expect((0, deviceClaims_1.sessionCap)(1)).to.equal(1);
+        });
+        it('is no limit when it is unset, zero, negative or not a number', () => {
+            for (const value of [undefined, null, 0, -1, NaN, Infinity, '8']) {
+                expect((0, deviceClaims_1.sessionCap)(value), String(value)).to.equal(undefined);
+            }
+        });
+    });
+    describe('allocation', () => {
+        (0, loki_stores_1.useLokiStores)();
+        const scratch = (0, scratch_database_1.useScratchDatabase)();
+        const sandbox = sinon_1.default.createSandbox();
+        const HOST = 'http://127.0.0.1:4723';
+        const pluginArgs = Object.assign({}, IPluginArgs_1.DefaultPluginArgs, {
+            iosDeviceType: 'both',
+            androidDeviceType: 'both',
+        });
+        const phone = (udid, over = {}) => (Object.assign({ udid, name: udid, platform: 'android', sdk: '13', host: HOST, busy: false, offline: false, userBlocked: false, state: 'device', deviceType: 'emulator', realDevice: false, systemPort: 57000, totalUtilizationTimeMilliSec: 0, sessionStartTime: 0 }, over));
+        // New each time: allocation pins the phone it picked into the caps it is given.
+        const caps = () => ({
+            alwaysMatch: { platformName: 'android' },
+            firstMatch: [{}],
+        });
+        const row = (udid) => __awaiter(void 0, void 0, void 0, function* () { return (yield db_1.XenonDatabase.DeviceModel).findOne({ udid }); });
+        /** Asks for a phone, waiting `ms` at most. Resolves to the udid, or null when held back. */
+        const ask = (ms = 400) => DeviceUtils.allocateDeviceForSession(caps(), ms, 50, pluginArgs).then((d) => d.udid, () => null);
+        const withCap = (maxSessions) => (0, test_container_1.createTestXenonManager)(Object.assign({}, pluginArgs, { maxSessions, platform: 'android' }));
+        before(() => __awaiter(void 0, void 0, void 0, function* () {
+            yield (0, test_container_1.resetTestContainer)();
+            yield (0, test_container_1.setupTestContainer)();
+        }));
+        after(() => (0, test_container_1.resetTestContainer)());
+        beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+            (yield db_1.XenonDatabase.DeviceModel).removeDataOnly();
+            yield scratch.db.lease.deleteMany({});
+            sandbox.stub(DeviceUtils, 'setUtilizationTime').resolves();
+        }));
+        afterEach(() => sandbox.restore());
+        const running = (n) => Array.from({ length: n }, (_, i) => phone(`running-${i}`, {
+            busy: true,
+            session_id: `sess-${i}`,
+            claimSessionId: `sess-${i}`,
+            claimedAt: 1,
+        }));
+        it('allocates while fewer sessions run than the limit', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(2);
+            yield (0, device_service_1.addNewDevice)([...running(1), phone('free-1')]);
+            expect(yield ask()).to.equal('free-1');
+        }));
+        it('holds a session back at the limit', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(2);
+            yield (0, device_service_1.addNewDevice)([...running(2), phone('free-1')]);
+            expect(yield ask()).to.equal(null);
+            expect((yield row('free-1')).busy).to.equal(false);
+        }));
+        it('still holds it back when more sessions run than the limit, which `===` let through', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(2);
+            yield (0, device_service_1.addNewDevice)([...running(3), phone('free-1')]);
+            expect(yield ask()).to.equal(null);
+            expect((yield row('free-1')).busy).to.equal(false);
+        }));
+        it('gives the phone once a session ends', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(2);
+            yield (0, device_service_1.addNewDevice)([...running(2), phone('free-1')]);
+            const waiting = ask(2000);
+            yield new Promise((r) => setTimeout(r, 150));
+            (yield db_1.XenonDatabase.DeviceModel)
+                .chain()
+                .find({ udid: 'running-0' })
+                .update((d) => Object.assign(d, {
+                busy: false,
+                session_id: null,
+                claimSessionId: null,
+                claimedAt: null,
+            }));
+            expect(yield waiting).to.be.oneOf(['free-1', 'running-0']);
+        }));
+        it('does not count live previews: they are not sessions', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(2);
+            yield (0, device_service_1.addNewDevice)([
+                phone('preview-1', { busy: true, session_id: 'manual_u1_preview-1' }),
+                phone('preview-2', { busy: true, session_id: 'manual_u1_preview-2' }),
+                phone('free-1'),
+            ]);
+            expect(yield ask()).to.equal('free-1');
+        }));
+        it('does not count a leased phone no session is running on', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(1);
+            yield (0, device_service_1.addNewDevice)([phone('leased-1', { busy: true }), phone('free-1')]);
+            yield scratch.db.lease.create({
+                data: {
+                    tokenHash: 'h',
+                    deviceUdid: 'leased-1',
+                    deviceHost: HOST,
+                    actorId: 'key_owner',
+                    status: 'active',
+                    expiresAt: Date.now() + 600000,
+                    lastHeartbeatAt: Date.now(),
+                    allocatedPorts: '{}',
+                    capabilityBag: '{}',
+                },
+            });
+            expect(yield ask()).to.equal('free-1');
+        }));
+        it('counts a session started on a leased phone', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(1);
+            yield (0, device_service_1.addNewDevice)([
+                phone('leased-1', {
+                    busy: true,
+                    session_id: 'sess-l',
+                    claimSessionId: 'sess-l',
+                    claimedAt: 1,
+                }),
+                phone('free-1'),
+            ]);
+            expect(yield ask()).to.equal(null);
+        }));
+        it('counts a session still being created', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(1);
+            yield (0, device_service_1.addNewDevice)([
+                phone('pending-1', { busy: true, claimedAt: Date.now() }),
+                phone('free-1'),
+            ]);
+            expect(yield ask()).to.equal(null);
+        }));
+        it('holds back exactly the sessions over the limit when many ask at once', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(2);
+            yield (0, device_service_1.addNewDevice)(Array.from({ length: 6 }, (_, i) => phone(`free-${i}`)));
+            const got = yield Promise.all(Array.from({ length: 5 }, () => ask(300)));
+            const given = got.filter((udid) => udid !== null);
+            expect(given, JSON.stringify(got)).to.have.length(2);
+            expect(new Set(given).size).to.equal(2);
+        }));
+        it('has no limit when the setting is not a positive number', () => __awaiter(void 0, void 0, void 0, function* () {
+            withCap(0);
+            yield (0, device_service_1.addNewDevice)([...running(3), phone('free-1')]);
+            expect(yield ask()).to.equal('free-1');
+        }));
+    });
+});

package/lib/test/unit/ocr-text-match.spec.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const chai_1 = require("chai");
+const ocrTextMatch_1 = require("../../src/services/omni-vision/ocrTextMatch");
+describe('ocrTextMatch', () => {
+    const word = (text, x0, y0, x1, y1, confidence = 90) => ({ text, x0, y0, x1, y1, confidence });
+    it('groups words into lines and runs, in reading order, despite a slightly uneven baseline', () => {
+        const runs = (0, ocrTextMatch_1.wordRuns)([
+            word('Done', 900, 40, 980, 70),
+            word('in', 158, 202, 180, 226),
+            word('Cancel', 20, 42, 120, 70),
+            word('Sign', 100, 200, 150, 224),
+        ]);
+        (0, chai_1.expect)(runs.map((r) => r.map((w) => w.text))).to.deep.equal([
+            ['Cancel'],
+            ['Done'],
+            ['Sign', 'in'],
+        ]);
+    });
+    it('finds every occurrence, once per set of words, with the lowest confidence of the words', () => {
+        const words = [
+            word('Add', 10, 10, 50, 30, 95),
+            word('to', 56, 10, 70, 30, 70),
+            word('cart', 76, 10, 120, 30, 88),
+            word('Add', 10, 100, 50, 120),
+            word('to', 56, 100, 70, 120),
+            word('cart', 76, 100, 120, 120),
+        ];
+        const found = (0, ocrTextMatch_1.findText)(words, '  add   TO cart ');
+        (0, chai_1.expect)(found.map((m) => [m.text, m.y0, m.confidence])).to.deep.equal([
+            ['Add to cart', 10, 70],
+            ['Add to cart', 100, 90],
+        ]);
+        (0, chai_1.expect)(found[0]).to.include({ x0: 10, x1: 120, y1: 30 });
+    });
+    it('matches a one-word text inside a word once, however often it appears there', () => {
+        (0, chai_1.expect)((0, ocrTextMatch_1.findText)([word('banana', 0, 0, 60, 20)], 'an').map((m) => m.text)).to.deep.equal([
+            'banana',
+        ]);
+    });
+    it('matches nothing for an empty text', () => {
+        (0, chai_1.expect)((0, ocrTextMatch_1.findText)([word('Sign', 0, 0, 40, 20)], '   ')).to.deep.equal([]);
+    });
+});

package/lib/test/unit/omni-vision-analyze-screen.spec.js

@@ -0,0 +1,72 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const AIService_1 = require("../../src/services/AIService");
+/**
+ * `xenon: analyzeScreen` (and device control's Omni-Scan) sent the screenshot's
+ * base64 where a file path was expected. The AI step found no such file and
+ * asked the provider without an image, under a "why did this test fail"
+ * prompt, so `ai_insights` was generic text about no screen at all.
+ *
+ * Now the screenshot goes to the provider with a prompt about the screen, and
+ * when there is no AI answer `ai_insights_error` says why.
+ */
+describe('xenon: analyzeScreen', () => {
+    const SCREEN = 'c2NyZWVuc2hvdA==';
+    let service;
+    const driver = { sessionId: 'session-1', getScreenshot: () => __awaiter(void 0, void 0, void 0, function* () { return SCREEN; }) };
+    beforeEach(() => {
+        service = new OmniVisionService_1.OmniVisionService();
+        sinon_1.default.stub(service, 'performOcr').resolves({
+            text: 'Sign in',
+            words: [{ text: 'Sign', confidence: 95, bbox: { x0: 10, y0: 10, x1: 50, y1: 30 } }],
+        });
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'initializeProvider');
+    });
+    afterEach(() => sinon_1.default.restore());
+    it('sends the screenshot to the AI provider and asks about the screen', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value({ analyze: () => __awaiter(void 0, void 0, void 0, function* () { return ''; }) });
+        const call = sinon_1.default
+            .stub(AIService_1.AI_SERVICE, 'callProvider')
+            .resolves('A sign-in screen with an email field.');
+        const analysis = yield service.analyzeScreen(driver);
+        (0, chai_1.expect)(call.calledOnce).to.equal(true);
+        const [prompt, image] = call.firstCall.args;
+        (0, chai_1.expect)(image).to.equal(SCREEN);
+        (0, chai_1.expect)(prompt).to.match(/screen/i);
+        (0, chai_1.expect)(prompt).to.not.match(/root cause|failed/i);
+        (0, chai_1.expect)(analysis.ai_insights).to.equal('A sign-in screen with an email field.');
+        (0, chai_1.expect)(analysis).to.not.have.property('ai_insights_error');
+        (0, chai_1.expect)(analysis.ocr.text).to.equal('Sign in');
+    }));
+    it('says why there are no AI insights when no provider is configured', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value(null);
+        const analysis = yield service.analyzeScreen(driver);
+        (0, chai_1.expect)(analysis.ai_insights).to.equal(null);
+        (0, chai_1.expect)(analysis.ai_insights_error).to.match(/No AI provider is configured/);
+        (0, chai_1.expect)(analysis.ocr.text).to.equal('Sign in');
+    }));
+    it('says why there are no AI insights when the provider fails', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value({ analyze: () => __awaiter(void 0, void 0, void 0, function* () { return ''; }) });
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'callProvider').rejects(new Error('429 quota exceeded'));
+        const analysis = yield service.analyzeScreen(driver);
+        (0, chai_1.expect)(analysis.ai_insights).to.equal(null);
+        (0, chai_1.expect)(analysis.ai_insights_error).to.match(/429 quota exceeded/);
+        (0, chai_1.expect)(analysis.ocr.text).to.equal('Sign in');
+    }));
+});

package/lib/test/unit/omni-vision-ios-points.spec.js

@@ -0,0 +1,161 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const sharp_1 = __importDefault(require("sharp"));
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const AIService_1 = require("../../src/services/AIService");
+const HealingOrchestrator_1 = require("../../src/services/healing/HealingOrchestrator");
+const types_1 = require("../../src/services/healing/types");
+const screenScale_1 = require("../../src/services/omni-vision/screenScale");
+/**
+ * OCR and the AI find a target in screenshot pixels. An iPhone's screenshot is
+ * 2x or 3x its screen in points, and XCUITest's W3C actions and element rects
+ * are in points, so a tap at the pixel position landed 2-3 times too far right
+ * and down. Nothing converted: not smartTap/visualTap, not the virtual
+ * elements the -custom:ai-* locators return, not the healing tiers' tap.
+ *
+ * Positions found in a screenshot are now converted to the driver's
+ * coordinates on iOS (screenshot size over window size). Android's W3C
+ * coordinates are the screenshot's pixels, so nothing changes there.
+ */
+describe('Omni-Vision taps in points on iOS', () => {
+    // A 3x screen: 30 x 60 pixels for a 10 x 20 point window.
+    let png3x;
+    before(() => __awaiter(void 0, void 0, void 0, function* () {
+        const image = yield (0, sharp_1.default)({
+            create: { width: 30, height: 60, channels: 3, background: { r: 255, g: 255, b: 255 } },
+        })
+            .png()
+            .toBuffer();
+        png3x = image.toString('base64');
+    }));
+    // "OK" at pixels 12..18 x 30..36: its middle is (15, 33) px, (5, 11) pt.
+    const OK_WORD = { text: 'OK', confidence: 90, bbox: { x0: 12, y0: 30, x1: 18, y1: 36 } };
+    let service;
+    let ios;
+    let android;
+    function driverFor(platformName) {
+        return {
+            sessionId: `session-${platformName}`,
+            caps: { platformName },
+            getScreenshot: sinon_1.default.stub().resolves(png3x),
+            getWindowRect: sinon_1.default.stub().resolves({ x: 0, y: 0, width: 10, height: 20 }),
+            performActions: sinon_1.default.stub().resolves(),
+            releaseActions: sinon_1.default.stub().resolves(),
+        };
+    }
+    beforeEach(() => {
+        service = new OmniVisionService_1.OmniVisionService();
+        sinon_1.default.stub(service, 'performOcr').resolves({ text: 'OK', words: [OK_WORD] });
+        ios = driverFor('iOS');
+        android = driverFor('Android');
+    });
+    afterEach(() => sinon_1.default.restore());
+    const tappedAt = (driver) => {
+        const move = driver.performActions.firstCall.args[0][0].actions[0];
+        return { x: move.x, y: move.y };
+    };
+    it('smartTap by text taps in points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        const result = yield service.omniClickByText(ios, { text: 'OK' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(tappedAt(ios)).to.deep.equal({ x: 5, y: 11 });
+        (0, chai_1.expect)(result.target).to.deep.include({
+            x: 5,
+            y: 11,
+            rect: { x: 4, y: 10, width: 2, height: 2 },
+        });
+    }));
+    it('smartTap by text taps in pixels on Android, without asking for the window', () => __awaiter(void 0, void 0, void 0, function* () {
+        const result = yield service.omniClickByText(android, { text: 'OK' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(tappedAt(android)).to.deep.equal({ x: 15, y: 33 });
+        (0, chai_1.expect)(android.getWindowRect.called).to.equal(false);
+    }));
+    it('visualTap taps in points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 15, y: 33 });
+        const result = yield service.omniClickByIcon(ios, { icon: 'the gear icon' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(tappedAt(ios)).to.deep.equal({ x: 5, y: 11 });
+    }));
+    it('gives the -custom:ai-text locator an element in points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        const [found] = yield service.findByText(ios, 'OK');
+        (0, chai_1.expect)(found.rect).to.deep.equal({ x: 4, y: 10, width: 2, height: 2 });
+        (0, chai_1.expect)((_a = service.getVirtualElement(found.id)) === null || _a === void 0 ? void 0 : _a.rect).to.deep.equal(found.rect);
+    }));
+    it("doesn't tap on iOS when the window size can't be read", () => __awaiter(void 0, void 0, void 0, function* () {
+        ios.getWindowRect.rejects(new Error('WDA is not running'));
+        const err = yield service.omniClickByText(ios, { text: 'OK' }).then(() => null, (e) => e);
+        (0, chai_1.expect)(err === null || err === void 0 ? void 0 : err.message).to.match(/WDA is not running/);
+        (0, chai_1.expect)(ios.performActions.called).to.equal(false);
+    }));
+    it('reads the window from getWindowSize when there is no getWindowRect', () => __awaiter(void 0, void 0, void 0, function* () {
+        const driver = {
+            caps: { platformName: 'iOS' },
+            getWindowSize: () => __awaiter(void 0, void 0, void 0, function* () { return ({ width: 15, height: 30 }); }),
+        };
+        (0, chai_1.expect)(yield (0, screenScale_1.screenScaleOf)(driver, png3x)).to.deep.equal({ x: 2, y: 2 });
+    }));
+    it("keeps pixels for a driver that isn't iOS, or can't be placed", () => __awaiter(void 0, void 0, void 0, function* () {
+        (0, chai_1.expect)(yield (0, screenScale_1.screenScaleOf)({}, 'not an image')).to.deep.equal({ x: 1, y: 1 });
+        (0, chai_1.expect)(yield (0, screenScale_1.screenScaleOf)({ caps: { platformName: 'Android' } }, 'x')).to.deep.equal({
+            x: 1,
+            y: 1,
+        });
+    }));
+    it('converts the OCR and Visual AI healing tiers’ rect to points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
+        const orchestrator = new HealingOrchestrator_1.HealingOrchestrator({
+            getSignature: () => __awaiter(void 0, void 0, void 0, function* () { return null; }),
+            saveSignature: () => __awaiter(void 0, void 0, void 0, function* () { return undefined; }),
+        });
+        const ocr = orchestrator.providers.find((p) => p.tier === types_1.HealingTier.TIER_3_LOCAL_OCR);
+        sinon_1.default.stub(ocr, 'heal').resolves({
+            id: 'healed_ocr_1',
+            tier: types_1.HealingTier.TIER_3_LOCAL_OCR,
+            confidence: 0.9,
+            originalSelector: '//*[@label="OK"]',
+            recommendedSelector: 'ocr:text="OK"',
+            rect: { x: 12, y: 30, width: 6, height: 6 },
+        });
+        ios.getPageSource = sinon_1.default.stub().resolves('<AppiumAUT/>');
+        const healed = yield orchestrator.attemptHealing('s1', ios, 'xpath', '//*[@label="OK"]', [3]);
+        (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.rect).to.deep.equal({ x: 4, y: 10, width: 2, height: 2 });
+    }));
+    it("drops a healed virtual element whose position on an iPhone can't be worked out", () => __awaiter(void 0, void 0, void 0, function* () {
+        const orchestrator = new HealingOrchestrator_1.HealingOrchestrator({
+            getSignature: () => __awaiter(void 0, void 0, void 0, function* () { return null; }),
+            saveSignature: () => __awaiter(void 0, void 0, void 0, function* () { return undefined; }),
+        });
+        const ocr = orchestrator.providers.find((p) => p.tier === types_1.HealingTier.TIER_3_LOCAL_OCR);
+        const found = {
+            tier: types_1.HealingTier.TIER_3_LOCAL_OCR,
+            confidence: 0.9,
+            originalSelector: '//*[@label="OK"]',
+            recommendedSelector: 'ocr:text="OK"',
+            rect: { x: 12, y: 30, width: 6, height: 6 },
+        };
+        const heal = sinon_1.default.stub(ocr, 'heal').resolves(Object.assign(Object.assign({}, found), { id: 'healed_ocr_1' }));
+        ios.getPageSource = sinon_1.default.stub().resolves('<AppiumAUT/>');
+        ios.getWindowRect.rejects(new Error('WDA is not running'));
+        (0, chai_1.expect)(yield orchestrator.attemptHealing('s1', ios, 'xpath', '//x', [3])).to.equal(null);
+        // A real element the tier resolved is kept, without the rect.
+        heal.resolves(Object.assign(Object.assign({}, found), { id: 'real-element-1' }));
+        const healed = yield orchestrator.attemptHealing('s1', ios, 'xpath', '//x', [3]);
+        (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.id).to.equal('real-element-1');
+        (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.rect).to.equal(undefined);
+    }));
+});

package/lib/test/unit/omni-vision-tap-text.spec.js

@@ -0,0 +1,129 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const AIService_1 = require("../../src/services/AIService");
+/**
+ * `xenon: smartTap { text }` (and `omniClick`) compared the text with one OCR
+ * word at a time, so any text with a space in it ("Sign in", "Forgot
+ * password?") never matched: the answer was `clicked: false`. The
+ * `-custom:ai-text` locator had the same rule. Now the text is looked for
+ * across neighbouring words on one line, and the tap goes to the middle of
+ * the words it covers.
+ */
+describe('Omni-Vision text taps match phrases', () => {
+    // Tesseract's words, in screenshot pixels. One line holds "Sign in" and,
+    // far to the right, "Help"; "Sign" and "in" also appear on two lines.
+    const word = (text, x0, y0, x1, y1) => ({
+        text,
+        confidence: 90,
+        bbox: { x0, y0, x1, y1 },
+    });
+    const WORDS = [
+        word('Sign', 100, 200, 150, 224),
+        word('in', 158, 200, 180, 224),
+        word('Help', 600, 200, 650, 224),
+        word('Forgot', 100, 300, 170, 324),
+        word('password?', 178, 300, 290, 324),
+        word('Sign', 100, 400, 150, 424),
+        word('in', 100, 440, 120, 464),
+    ];
+    let service;
+    let driver;
+    beforeEach(() => {
+        service = new OmniVisionService_1.OmniVisionService();
+        sinon_1.default.stub(service, 'performOcr').resolves({ text: '', words: WORDS });
+        driver = {
+            sessionId: 'session-1',
+            // An Android driver: its W3C coordinates are the screenshot's pixels.
+            caps: { platformName: 'Android' },
+            getScreenshot: sinon_1.default.stub().resolves('aGVsbG8='),
+            performActions: sinon_1.default.stub().resolves(),
+            releaseActions: sinon_1.default.stub().resolves(),
+        };
+    });
+    afterEach(() => sinon_1.default.restore());
+    const tappedAt = () => {
+        const move = driver.performActions.firstCall.args[0][0].actions[0];
+        return { x: move.x, y: move.y };
+    };
+    it('taps the middle of a two-word text', () => __awaiter(void 0, void 0, void 0, function* () {
+        const result = yield service.omniClickByText(driver, { text: 'Sign in' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(result.target).to.deep.include({
+            text: 'Sign in',
+            x: 140,
+            y: 212,
+            rect: { x: 100, y: 200, width: 80, height: 24 },
+        });
+        (0, chai_1.expect)(tappedAt()).to.deep.equal({ x: 140, y: 212 });
+    }));
+    it('matches a phrase whatever its case, and part of a word at either end', () => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        const result = yield service.omniClickByText(driver, { text: 'forgot PASSWORD' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)((_a = result.target) === null || _a === void 0 ? void 0 : _a.text).to.equal('Forgot password?');
+        (0, chai_1.expect)(tappedAt()).to.deep.equal({ x: 195, y: 312 });
+    }));
+    it('does not join words on different lines, or far apart on one line', () => __awaiter(void 0, void 0, void 0, function* () {
+        for (const text of ['in Help', 'Sign in Help', 'password? Sign']) {
+            const result = yield service.omniClickByText(driver, { text });
+            (0, chai_1.expect)(result.clicked, text).to.equal(false);
+        }
+        (0, chai_1.expect)(driver.performActions.called).to.equal(false);
+    }));
+    it('still matches a single word, as before', () => __awaiter(void 0, void 0, void 0, function* () {
+        const result = yield service.omniClickByText(driver, { text: 'help' });
+        (0, chai_1.expect)(result.clicked, result.message).to.equal(true);
+        (0, chai_1.expect)(tappedAt()).to.deep.equal({ x: 625, y: 212 });
+    }));
+    it('finds a phrase with the -custom:ai-text locator too', () => __awaiter(void 0, void 0, void 0, function* () {
+        const found = yield service.findByText(driver, 'Sign in');
+        (0, chai_1.expect)(found).to.have.lengthOf(1);
+        (0, chai_1.expect)(found[0].text).to.equal('Sign in');
+        (0, chai_1.expect)(found[0].rect).to.deep.equal({ x: 100, y: 200, width: 80, height: 24 });
+    }));
+});
+/**
+ * `xenon: smartTap { icon }` / `visualTap` answered `clicked: false`, "No visual
+ * match found", when it never looked: no AI provider, or the AI call failed.
+ * It now fails with the reason; `clicked: false` means the AI looked and found
+ * nothing.
+ */
+describe('Omni-Vision icon taps that cannot look', () => {
+    const driver = {
+        sessionId: 'session-1',
+        caps: { platformName: 'Android' },
+        getScreenshot: () => __awaiter(void 0, void 0, void 0, function* () { return 'aGVsbG8='; }),
+        performActions: sinon_1.default.stub().resolves(),
+    };
+    beforeEach(() => {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'initializeProvider');
+    });
+    afterEach(() => sinon_1.default.restore());
+    it('fails when no AI provider is configured', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value(null);
+        const err = yield new OmniVisionService_1.OmniVisionService().omniClickByIcon(driver, { icon: 'the gear' }).then(() => null, (e) => e);
+        (0, chai_1.expect)(err === null || err === void 0 ? void 0 : err.message).to.match(/No AI provider is configured/);
+    }));
+    it('answers clicked: false when the AI looked and found nothing', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves(null);
+        const result = yield new OmniVisionService_1.OmniVisionService().omniClickByIcon(driver, { icon: 'the gear' });
+        (0, chai_1.expect)(result.clicked).to.equal(false);
+        (0, chai_1.expect)(result.message).to.match(/No visual match/);
+    }));
+});

package/lib/test/unit/omni-vision.spec.js

@@ -95,7 +95,7 @@ describe('OmniVisionService Unit Tests', () => {
             terminate: sinon_1.default.stub().resolves(),
         };
         sinon_1.default.stub(tesseract_js_1.default, 'createWorker').resolves(mockWorker);
-        sinon_1.default.stub(AIService_1.AI_SERVICE, 'analyzeFailure').resolves('AI Insights text');
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'describeScreen').resolves('AI Insights text');
         const analysis = yield omniService.analyzeScreen(mockDriver);
         (0, chai_1.expect)(analysis.ocr.text).to.equal('Screen Text');
         (0, chai_1.expect)(analysis.ai_insights).to.equal('AI Insights text');