@xenon-device-management/xenon 2.14.0 → 2.16.0

package/lib/test/unit/ocr-healing-tier.spec.js

@@ -0,0 +1,122 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const OcrHealingProvider_1 = require("../../src/services/healing/OcrHealingProvider");
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const VisualAiHealingProvider_1 = require("../../src/services/healing/VisualAiHealingProvider");
+const AIService_1 = require("../../src/services/AIService");
+/**
+ * Self-healing's OCR tier took the first word OCR read that contained the
+ * text, or was part of it: for `Sign in` that was any lone "in" or "Sign" on
+ * the screen, and for `Login` a "Log". It now matches as Omni-Vision's
+ * `-custom:ai-text` does (ocrTextMatch.ts): the text across neighbouring words
+ * on one line, the first match in reading order.
+ *
+ * OCR is stubbed with the words it would read, in screenshot pixels: no
+ * Tesseract runs. ocr-healing-tier-real-ocr.spec.ts runs the tier on a real
+ * screenshot, which a stub can't stand in for: tesseract.js's own output is
+ * what hid the tier's never healing.
+ */
+describe('The OCR healing tier matches text as Omni-Vision does', () => {
+    const word = (text, x0, y0, x1, y1) => ({
+        text,
+        confidence: 90,
+        bbox: { x0, y0, x1, y1 },
+    });
+    // A heading with "Log" and a lone "in" read first; then "Sign in", far
+    // from "Help" on its line; then "Login".
+    const WORDS = [
+        word('Log', 40, 60, 90, 84),
+        word('in', 300, 60, 320, 84),
+        word('Sign', 100, 200, 150, 224),
+        word('in', 158, 200, 180, 224),
+        word('Help', 600, 200, 650, 224),
+        word('Login', 100, 400, 180, 424),
+    ];
+    let driver;
+    beforeEach(() => {
+        // OCR reads these words: the tier asks Omni-Vision's OCR.
+        sinon_1.default
+            .stub(typedi_1.Container.get(OmniVisionService_1.OmniVisionService), 'performOcr')
+            .resolves({ text: '', words: WORDS });
+        // Android: no element is found by an iOS predicate, so the tier answers
+        // with where the text is.
+        driver = { findElement: sinon_1.default.stub().rejects(new Error('unsupported locator strategy')) };
+    });
+    afterEach(() => sinon_1.default.restore());
+    const heal = (selector) => new OcrHealingProvider_1.OcrHealingProvider().heal({
+        sessionId: 'ocr-session',
+        driver,
+        strategy: 'xpath',
+        selector,
+        screenshotBase64: 'aGVsbG8=',
+    });
+    it('finds a two-word text across its words, not a lone word of it', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='Sign in']");
+        (0, chai_1.expect)(healed, 'healed').to.not.equal(null);
+        (0, chai_1.expect)(healed.recommendedSelector).to.equal('ocr:text="Sign in"');
+        (0, chai_1.expect)(healed.rect).to.deep.equal({ x: 100, y: 200, width: 80, height: 24 });
+    }));
+    it('hands on the text it read, for getText on the element it returns', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='sign IN']");
+        (0, chai_1.expect)(healed.text).to.equal('Sign in');
+        (0, chai_1.expect)(healed.message).to.not.equal(healed.text);
+    }));
+    it('does not take a word that is only part of the text', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='Login']");
+        (0, chai_1.expect)(healed, 'healed').to.not.equal(null);
+        (0, chai_1.expect)(healed.recommendedSelector).to.equal('ocr:text="Login"');
+        (0, chai_1.expect)(healed.rect).to.deep.equal({ x: 100, y: 400, width: 80, height: 24 });
+    }));
+    it('finds nothing when the text is not on the screen', () => __awaiter(void 0, void 0, void 0, function* () {
+        (0, chai_1.expect)(yield heal("//*[@text='Sign up']")).to.equal(null);
+    }));
+});
+/**
+ * A healed element's id names it in the server's memory of virtual elements,
+ * which every session shares. `healed_ocr_<ms>` was the same for two heals in
+ * one millisecond, so the second overwrote the first.
+ */
+describe('Elements healed by OCR or Visual AI', () => {
+    afterEach(() => sinon_1.default.restore());
+    const context = {
+        sessionId: 'ids-session',
+        driver: { findElement: () => __awaiter(void 0, void 0, void 0, function* () { return Promise.reject(new Error('no predicate')); }) },
+        strategy: 'xpath',
+        selector: "//*[@text='Login']",
+        screenshotBase64: 'aGVsbG8=',
+    };
+    it('get ids of their own when healed in the same millisecond', () => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, _b, _c, _d;
+        sinon_1.default.stub(Date, 'now').returns(1700000000000);
+        sinon_1.default.stub(typedi_1.Container.get(OmniVisionService_1.OmniVisionService), 'performOcr').resolves({
+            text: '',
+            words: [{ text: 'Login', confidence: 90, bbox: { x0: 1, y0: 1, x1: 50, y1: 20 } }],
+        });
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'isEnabled').returns(true);
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 100, y: 100 });
+        const ids = [
+            (_a = (yield new OcrHealingProvider_1.OcrHealingProvider().heal(context))) === null || _a === void 0 ? void 0 : _a.id,
+            (_b = (yield new OcrHealingProvider_1.OcrHealingProvider().heal(context))) === null || _b === void 0 ? void 0 : _b.id,
+            (_c = (yield new VisualAiHealingProvider_1.VisualAiHealingProvider().heal(context))) === null || _c === void 0 ? void 0 : _c.id,
+            (_d = (yield new VisualAiHealingProvider_1.VisualAiHealingProvider().heal(context))) === null || _d === void 0 ? void 0 : _d.id,
+        ];
+        (0, chai_1.expect)(ids.every(Boolean), String(ids)).to.equal(true);
+        (0, chai_1.expect)(new Set(ids).size, String(ids)).to.equal(4);
+    }));
+});

package/lib/test/unit/ocr-offline.spec.js

@@ -0,0 +1,213 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = __importDefault(require("fs"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const sharp_1 = __importDefault(require("sharp"));
+const sinon_1 = __importDefault(require("sinon"));
+const tesseract_js_1 = __importDefault(require("tesseract.js"));
+const typedi_1 = require("typedi");
+const package_json_1 = __importDefault(require("../../package.json"));
+const ocrData_1 = require("../../src/services/ocr/ocrData");
+const OcrHealingProvider_1 = require("../../src/services/healing/OcrHealingProvider");
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const container_registration_1 = require("../helpers/container-registration");
+/**
+ * tesseract.js downloads its English data (eng.traineddata) from
+ * cdn.jsdelivr.net the first time a worker starts, and caches it in the
+ * server's working directory. A server with no internet couldn't run OCR at
+ * all: the first OCR call never finished, and every later one waited for it.
+ * One with internet left a 5 MB eng.traineddata wherever it was started (the
+ * home directory, ~/.appium).
+ *
+ * The data now ships with the plugin, read from there and cached nowhere.
+ */
+describe('OCR without the internet', () => {
+    // The upstream file, @tesseract.js-data/eng@1.0.0 4.0.0_best_int/eng.traineddata.gz:
+    // what tesseract.js 7 downloads by default (LSTM only).
+    const UPSTREAM_SHA256 = '45b4cb346724ac1774f1c36f42f182b887bcdb28ebe63e6fff90ac41f3fcff91';
+    const tmpDirs = [];
+    const tmpDir = () => {
+        const dir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), 'xenon-ocr-'));
+        tmpDirs.push(dir);
+        return dir;
+    };
+    afterEach(() => {
+        sinon_1.default.restore();
+        for (const dir of tmpDirs.splice(0))
+            fs_1.default.rmSync(dir, { recursive: true, force: true });
+    });
+    /** A white image with `text` on it, as a phone's screenshot would have. */
+    const imageOf = (text) => (0, sharp_1.default)({ create: { width: 900, height: 220, channels: 3, background: '#ffffff' } })
+        .composite([
+        {
+            input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="900" height="220">` +
+                `<text x="40" y="140" font-family="Helvetica, Arial, sans-serif" font-size="96" fill="#000">${text}</text>` +
+                `</svg>`),
+        },
+    ])
+        .png()
+        .toBuffer();
+    describe('the English data', () => {
+        it('ships with the plugin, byte for byte the file tesseract.js would download', () => {
+            (0, chai_1.expect)(ocrData_1.OCR_DATA_FILE).to.equal(path_1.default.join(ocrData_1.OCR_DATA_DIR, 'eng.traineddata.gz'));
+            const sha = crypto_1.default.createHash('sha256').update(fs_1.default.readFileSync(ocrData_1.OCR_DATA_FILE)).digest('hex');
+            (0, chai_1.expect)(sha).to.equal(UPSTREAM_SHA256);
+            (0, chai_1.expect)(ocrData_1.OCR_DATA_SHA256).to.equal(UPSTREAM_SHA256);
+            const readme = fs_1.default.readFileSync(path_1.default.join(ocrData_1.OCR_DATA_DIR, 'README.md'), 'utf8');
+            (0, chai_1.expect)(readme).to.include(UPSTREAM_SHA256);
+        });
+        it('is copied into lib/ by the build, which is what the npm package ships', () => {
+            const copy = package_json_1.default.scripts['build:copy'];
+            (0, chai_1.expect)(copy).to.include('mkdir -p lib/src/services/ocr/vendor');
+            (0, chai_1.expect)(copy).to.include('cp src/services/ocr/vendor/*.gz');
+            (0, chai_1.expect)(package_json_1.default.files).to.include('lib');
+        });
+        it('is read from there, never fetched, and cached nowhere', () => {
+            const options = (0, ocrData_1.ocrWorkerOptions)();
+            (0, chai_1.expect)(options.langPath).to.equal(ocrData_1.OCR_DATA_DIR);
+            (0, chai_1.expect)(String(options.langPath)).not.to.match(/^[a-z]+:\/\//i);
+            (0, chai_1.expect)(options.cacheMethod).to.equal('none');
+            (0, chai_1.expect)(options.gzip).to.equal(true);
+        });
+    });
+    describe('a real OCR worker', () => {
+        let cwd;
+        beforeEach(() => {
+            cwd = process.cwd();
+        });
+        afterEach(() => {
+            process.chdir(cwd);
+        });
+        it('reads text, and writes nothing into the working directory', () => __awaiter(void 0, void 0, void 0, function* () {
+            const workDir = tmpDir();
+            process.chdir(workDir);
+            const worker = yield (0, ocrData_1.createOcrWorker)();
+            try {
+                const { data } = yield worker.recognize(yield imageOf('Sign in'));
+                (0, chai_1.expect)(data.text).to.match(/sign\s*in/i);
+            }
+            finally {
+                yield worker.terminate();
+            }
+            (0, chai_1.expect)(fs_1.default.readdirSync(workDir)).to.deep.equal([]);
+        }));
+        it('fails with a reason, rather than wait for ever, when the data is missing', () => __awaiter(void 0, void 0, void 0, function* () {
+            const started = Date.now();
+            const failure = yield (0, ocrData_1.createOcrWorker)(tmpDir()).then(() => chai_1.expect.fail('a worker with no data should not start'), (err) => err);
+            (0, chai_1.expect)(failure.message).to.include('eng.traineddata.gz');
+            (0, chai_1.expect)(Date.now() - started).to.be.below(5000);
+        }));
+        // Given data it can't load, tesseract.js 7 throws from its own message
+        // handler, and the server exits on an uncaught exception: the data is
+        // checked before a worker sees it.
+        for (const [damage, bytes] of [
+            ['not tesseract data at all', () => Buffer.from('not tesseract data')],
+            ['cut short', () => fs_1.default.readFileSync(ocrData_1.OCR_DATA_FILE).subarray(0, 100000)],
+        ]) {
+            it(`fails with a reason, before any worker starts, when the data is ${damage}`, () => __awaiter(void 0, void 0, void 0, function* () {
+                const dir = tmpDir();
+                fs_1.default.writeFileSync(path_1.default.join(dir, 'eng.traineddata.gz'), bytes());
+                const createWorker = sinon_1.default.spy(tesseract_js_1.default, 'createWorker');
+                const failure = yield (0, ocrData_1.createOcrWorker)(dir).then(() => chai_1.expect.fail('a worker with damaged data should not start'), (err) => err);
+                (0, chai_1.expect)(failure.message).to.include('damaged');
+                (0, chai_1.expect)(createWorker.called).to.equal(false);
+            }));
+        }
+    });
+    describe('a worker that will not start', () => {
+        /** A directory holding a good copy of the data, so this test's memory is its own. */
+        const dataDir = () => {
+            const dir = tmpDir();
+            fs_1.default.copyFileSync(ocrData_1.OCR_DATA_FILE, path_1.default.join(dir, 'eng.traineddata.gz'));
+            return dir;
+        };
+        it('is remembered for a minute: OCR fails at once rather than start another', () => __awaiter(void 0, void 0, void 0, function* () {
+            // tesseract.js hands back no worker when its start fails, so its thread
+            // can't be ended; starting one per OCR call would leave one per call.
+            const clock = sinon_1.default.useFakeTimers({ now: Date.now(), toFake: ['Date'] });
+            const createWorker = sinon_1.default
+                .stub(tesseract_js_1.default, 'createWorker')
+                .callsFake((_langs, _oem, options) => {
+                setTimeout(() => options.errorHandler('the OCR engine failed to load'), 5);
+                return new Promise(() => undefined);
+            });
+            const dir = dataDir();
+            const reason = (p) => p.then(() => chai_1.expect.fail('the worker should not start'), (err) => err.message);
+            (0, chai_1.expect)(yield reason((0, ocrData_1.createOcrWorker)(dir))).to.include('the OCR engine failed to load');
+            (0, chai_1.expect)(yield reason((0, ocrData_1.createOcrWorker)(dir))).to.include('the OCR engine failed to load');
+            (0, chai_1.expect)(createWorker.callCount).to.equal(1);
+            clock.tick(60001);
+            yield reason((0, ocrData_1.createOcrWorker)(dir));
+            (0, chai_1.expect)(createWorker.callCount).to.equal(2);
+        }));
+    });
+    describe('every place that runs OCR', () => {
+        let restoreContainer;
+        beforeEach(() => {
+            restoreContainer = (0, container_registration_1.saveRegistrations)(OmniVisionService_1.OmniVisionService);
+        });
+        afterEach(() => {
+            restoreContainer();
+        });
+        it("the OCR healing tier reads the shipped data, with Omni-Vision's worker", () => __awaiter(void 0, void 0, void 0, function* () {
+            // The tier reads the screenshot through Omni-Vision (findTextInScreenshot),
+            // so it shares that worker and leaves it running for the next read.
+            const worker = {
+                setParameters: sinon_1.default.stub().resolves(),
+                recognize: sinon_1.default.stub().resolves({ data: { text: '', words: [], hocr: '', tsv: '' } }),
+                terminate: sinon_1.default.stub().resolves(),
+            };
+            const createWorker = sinon_1.default.stub(tesseract_js_1.default, 'createWorker').resolves(worker);
+            typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+            yield new OcrHealingProvider_1.OcrHealingProvider().heal({
+                screenshotBase64: (yield imageOf('Login')).toString('base64'),
+                selector: "//*[@text='Login']",
+                strategy: 'xpath',
+                driver: {},
+            });
+            (0, chai_1.expect)(createWorker.calledOnce).to.equal(true);
+            (0, chai_1.expect)(createWorker.firstCall.args[0]).to.equal('eng');
+            (0, chai_1.expect)(createWorker.firstCall.args[2]).to.include({
+                langPath: ocrData_1.OCR_DATA_DIR,
+                cacheMethod: 'none',
+            });
+            (0, chai_1.expect)(worker.recognize.calledOnce).to.equal(true);
+            (0, chai_1.expect)(worker.terminate.called).to.equal(false);
+        }));
+        it("Omni-Vision's worker reads the shipped data", () => __awaiter(void 0, void 0, void 0, function* () {
+            const worker = {
+                setParameters: sinon_1.default.stub().resolves(),
+                recognize: sinon_1.default.stub().resolves({ data: { text: '', words: [], hocr: '', tsv: '' } }),
+                terminate: sinon_1.default.stub().resolves(),
+            };
+            const createWorker = sinon_1.default.stub(tesseract_js_1.default, 'createWorker').resolves(worker);
+            typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+            const driver = {
+                getScreenshot: sinon_1.default.stub().resolves((yield imageOf('Login')).toString('base64')),
+            };
+            yield typedi_1.Container.get(OmniVisionService_1.OmniVisionService).findByText(driver, 'Login');
+            (0, chai_1.expect)(createWorker.calledOnce).to.equal(true);
+            (0, chai_1.expect)(createWorker.firstCall.args[0]).to.equal('eng');
+            (0, chai_1.expect)(createWorker.firstCall.args[2]).to.include({
+                langPath: ocrData_1.OCR_DATA_DIR,
+                cacheMethod: 'none',
+            });
+        }));
+    });
+});

package/lib/test/unit/omni-virtual-element-store.spec.js

@@ -0,0 +1,301 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const plugin_1 = require("../../src/plugin");
+const NodeDevices_1 = __importDefault(require("../../src/device-managers/NodeDevices"));
+const DeviceService = __importStar(require("../../src/data-service/device-service"));
+const ActiveLeases = __importStar(require("../../src/services/lease/activeLeases"));
+const device_utils_1 = require("../../src/device-utils");
+const CommandInterceptor_1 = require("../../src/interceptors/CommandInterceptor");
+const HealingOrchestrator_1 = require("../../src/services/healing/HealingOrchestrator");
+const types_1 = require("../../src/services/healing/types");
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const AutowaitService_1 = require("../../src/services/autowait/AutowaitService");
+const SessionLifecycleService_1 = require("../../src/services/SessionLifecycleService");
+const SessionMetricsService_1 = require("../../src/services/metrics/SessionMetricsService");
+const OrphanSweeper_1 = require("../../src/services/OrphanSweeper");
+const PhoneNetworkRestore_1 = require("../../src/services/network/PhoneNetworkRestore");
+const SelfHealingSwitch_1 = require("../../src/services/settings/SelfHealingSwitch");
+const AIService_1 = require("../../src/services/AIService");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * An element Xenon finds in a screenshot (a `-custom:ai-*` find, or an OCR or
+ * Visual AI heal) is a box on the screen it remembers by id. Through 2.14 it
+ * remembered every one, from every session, until the server restarted:
+ * device control's locator test and an icon `smartTap` added one per call
+ * that nothing ever read, and any session could act on another's. Now an
+ * element belongs to the session that found it and goes when that session
+ * ends, and the store has a size limit.
+ *
+ * OCR, the AI provider and the drivers are stubbed: no Tesseract, no AI call,
+ * no phone.
+ */
+const W3C = 'element-6066-11e4-a52e-4f735466cecf';
+const WORDS = [
+    { text: 'Sign', confidence: 90, bbox: { x0: 100, y0: 200, x1: 150, y1: 224 } },
+    { text: 'in', confidence: 90, bbox: { x0: 158, y0: 200, x1: 180, y1: 224 } },
+];
+const box = (id) => ({
+    id,
+    rect: { x: 0, y: 0, width: 10, height: 10 },
+    confidence: 0.9,
+});
+describe('Elements found in a screenshot belong to their session', () => {
+    (0, scratch_database_1.useScratchDatabase)();
+    let restoreContainer;
+    let omni;
+    const driverFor = (sessionId) => ({
+        sessionId,
+        caps: { platformName: 'Android' },
+        getScreenshot: sinon_1.default.stub().resolves('aGVsbG8='),
+        performActions: sinon_1.default.stub().resolves(),
+        releaseActions: sinon_1.default.stub().resolves(),
+    });
+    beforeEach(() => {
+        restoreContainer = (0, container_registration_1.saveRegistrations)(OmniVisionService_1.OmniVisionService, HealingOrchestrator_1.HealingOrchestrator, SelfHealingSwitch_1.SelfHealingSwitch);
+        typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+        typedi_1.Container.set(SelfHealingSwitch_1.SelfHealingSwitch, new SelfHealingSwitch_1.SelfHealingSwitch());
+        omni = typedi_1.Container.get(OmniVisionService_1.OmniVisionService);
+        sinon_1.default.stub(omni, 'performOcr').resolves({ text: '', words: WORDS });
+        sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'afterSessionCommand').resolves();
+    });
+    afterEach(() => {
+        restoreContainer();
+        sinon_1.default.restore();
+    });
+    const run = (driver, command, args, next = () => null) => typedi_1.Container.get(CommandInterceptor_1.CommandInterceptor).handle(() => __awaiter(void 0, void 0, void 0, function* () { return next(); }), driver, command, [...args, driver.sessionId], Object.assign({}, IPluginArgs_1.DefaultPluginArgs), false);
+    const failure = (promise) => __awaiter(void 0, void 0, void 0, function* () {
+        try {
+            yield promise;
+        }
+        catch (err) {
+            return err;
+        }
+        throw new Error('expected the command to fail');
+    });
+    const held = () => omni.elements.size;
+    it("answers another session's command on it with no such element", () => __awaiter(void 0, void 0, void 0, function* () {
+        const a = driverFor('sess-a');
+        const b = driverFor('sess-b');
+        const found = yield run(a, 'findElement', ['-custom:ai-text', 'Sign in']);
+        const err = yield failure(run(b, 'click', [found[W3C]]));
+        (0, chai_1.expect)(err.error).to.equal('no such element');
+        (0, chai_1.expect)(b.performActions.called, "tapped on the other session's phone").to.equal(false);
+        yield run(a, 'click', [found[W3C]]);
+        (0, chai_1.expect)(a.performActions.calledOnce).to.equal(true);
+    }));
+    it('keeps a healed element for the session it healed in', () => __awaiter(void 0, void 0, void 0, function* () {
+        const attemptHealing = sinon_1.default.stub().resolves({
+            id: 'healed_ocr_1_abc',
+            tier: types_1.HealingTier.TIER_3_LOCAL_OCR,
+            confidence: 0.9,
+            originalSelector: "//*[@text='Sign in']",
+            originalStrategy: 'xpath',
+            recommendedSelector: 'ocr:text="Sign in"',
+            recommendedStrategy: 'xenon:visual',
+            text: 'Sign in',
+            rect: { x: 100, y: 200, width: 80, height: 24 },
+        });
+        typedi_1.Container.set(HealingOrchestrator_1.HealingOrchestrator, { attemptHealing });
+        const a = driverFor('sess-a');
+        yield run(a, 'findElement', ['xpath', "//*[@text='Sign in']"], () => {
+            throw Object.assign(new Error('NoSuchElement'), { name: 'NoSuchElementError' });
+        });
+        (0, chai_1.expect)(yield run(a, 'getText', ['healed_ocr_1_abc'])).to.equal('Sign in');
+        const err = yield failure(run(driverFor('sess-b'), 'getText', ['healed_ocr_1_abc']));
+        (0, chai_1.expect)(err.error).to.equal('no such element');
+    }));
+    it("keeps nothing for device control's locator test", () => __awaiter(void 0, void 0, void 0, function* () {
+        // As POST /control/<udid>/test-locator calls it: the ids are only shown.
+        const manual = { sessionId: 'manual_R5CT', getScreenshot: sinon_1.default.stub().resolves('aGVsbG8=') };
+        const found = yield omni.findByText(manual, 'Sign in', { throwOnError: true });
+        (0, chai_1.expect)(found).to.have.length(1);
+        (0, chai_1.expect)(held()).to.equal(0);
+    }));
+    it('keeps nothing for an icon smartTap, which taps at once', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 50, y: 50 });
+        const result = yield omni.omniClickByIcon(driverFor('sess-a'), { icon: 'the cart' });
+        (0, chai_1.expect)(result.clicked).to.equal(true);
+        (0, chai_1.expect)(held()).to.equal(0);
+    }));
+    it('keeps the newest 1,000 for a session', () => {
+        for (let i = 0; i < 1001; i++)
+            omni.remember('sess-a', box(`omni_ocr_${i}`));
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_0', 'sess-a')).to.equal(undefined);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_1', 'sess-a')).to.not.equal(undefined);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_1000', 'sess-a')).to.not.equal(undefined);
+    });
+    it('keeps the newest 10,000 in all', () => {
+        for (let s = 0; s < 11; s++) {
+            for (let i = 0; i < 1000; i++)
+                omni.remember(`sess-${s}`, box(`omni_ocr_${s}_${i}`));
+        }
+        (0, chai_1.expect)(held()).to.equal(10000);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_0_999', 'sess-0')).to.equal(undefined);
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_1_0', 'sess-1')).to.not.equal(undefined);
+    });
+    it('gives icon matches ids of their own, as text matches have', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(Date, 'now').returns(1700000000000);
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 50, y: 50 });
+        const a = driverFor('sess-a');
+        const first = yield omni.findByIcon(a, 'the cart');
+        const second = yield omni.findByIcon(a, 'the cart');
+        (0, chai_1.expect)(first === null || first === void 0 ? void 0 : first.id).to.not.equal(second === null || second === void 0 ? void 0 : second.id);
+    }));
+});
+/**
+ * Everything a session leaves in this server's memory for its commands goes
+ * when the session ends, however it ends: its virtual elements, and its
+ * autowait settings (`xenon: setAutowaitProperties`). Those were meant to be
+ * cleared by the interceptor on `deleteSession`, but XenonPlugin answers
+ * `deleteSession` itself, so Appium never hands that command to the
+ * interceptor: they were kept for good.
+ */
+describe("A session's virtual elements and autowait settings go when it ends", function () {
+    this.timeout(30000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let restoreRegs;
+    let omni;
+    const seed = (sessionId) => {
+        omni.remember(sessionId, box(`omni_ocr_${sessionId}`));
+        typedi_1.Container.get(AutowaitService_1.AutowaitService).setProps(sessionId, { timeoutMs: 1234 });
+    };
+    const expectForgotten = (sessionId) => {
+        (0, chai_1.expect)(omni.getVirtualElement(`omni_ocr_${sessionId}`, sessionId), 'element').to.equal(undefined);
+        (0, chai_1.expect)(typedi_1.Container.get(AutowaitService_1.AutowaitService).getProps(sessionId, Object.assign({}, IPluginArgs_1.DefaultPluginArgs)).timeoutMs, 'autowait timeout').to.equal(IPluginArgs_1.DefaultPluginArgs.autowait.timeoutMs);
+    };
+    beforeEach(() => {
+        restoreRegs = (0, container_registration_1.saveRegistrations)(OmniVisionService_1.OmniVisionService, AutowaitService_1.AutowaitService, PhoneNetworkRestore_1.PhoneNetworkRestore, SessionMetricsService_1.SessionMetricsService);
+        typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+        typedi_1.Container.set(AutowaitService_1.AutowaitService, new AutowaitService_1.AutowaitService());
+        omni = typedi_1.Container.get(OmniVisionService_1.OmniVisionService);
+        typedi_1.Container.set(PhoneNetworkRestore_1.PhoneNetworkRestore, {
+            restoreSession: sinon_1.default.stub().resolves(),
+            restoreAll: sinon_1.default.stub().resolves(),
+        });
+        typedi_1.Container.set(SessionMetricsService_1.SessionMetricsService, { stop: sinon_1.default.stub().resolves() });
+        sinon_1.default.stub(DeviceService, 'releaseSessionDevices').resolves();
+        sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'onSessionStopped').resolves();
+    });
+    afterEach(() => {
+        sinon_1.default.restore();
+        restoreRegs();
+    });
+    it('when the test deletes the session', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-deleted');
+        yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).deleteSession(() => __awaiter(this, void 0, void 0, function* () { return null; }), 's-deleted');
+        expectForgotten('s-deleted');
+    }));
+    it('when Appium ends the session itself (new-command timeout)', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-timeout');
+        const plugin = { pluginArgs: {}, xenonLog: { withSession: () => ({ info() { } }) } };
+        yield plugin_1.XenonPlugin.prototype.onUnexpectedShutdown.call(plugin, { sessionId: 's-timeout', caps: { udid: 'R5CT' } }, new Error('New command timeout'));
+        expectForgotten('s-timeout');
+    }));
+    it('when Appium ends the session on a node', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-node');
+        sinon_1.default.stub(NodeDevices_1.default.prototype, 'unblockDevice').resolves(undefined);
+        const plugin = { pluginArgs: { hub: 'http://hub:4723' } };
+        yield plugin_1.XenonPlugin.prototype.onUnexpectedShutdown.call(plugin, { sessionId: 's-node', caps: {} }, null);
+        expectForgotten('s-node');
+    }));
+    it("when Xenon's idle check releases the phone", () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-idle');
+        const idle = {
+            udid: 'R5CT',
+            host: 'http://127.0.0.1:4723',
+            busy: true,
+            userBlocked: false,
+            session_id: 's-idle',
+            claimSessionId: 's-idle',
+            lastCmdExecutedAt: Date.now() - 10 * 60 * 1000,
+        };
+        sinon_1.default.stub(DeviceService, 'getAllDevices').resolves([idle]);
+        sinon_1.default.stub(ActiveLeases, 'activeLeasesByDevice').resolves(new Map());
+        sinon_1.default.stub(DeviceService, 'releaseSessionDevice').resolves(true);
+        yield (0, device_utils_1.releaseBlockedDevices)(60);
+        expectForgotten('s-idle');
+    }));
+    it('when a graceful shutdown drains the session', () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-down');
+        yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).stopSessionForShutdown('s-down', 'shutdown');
+        expectForgotten('s-down');
+    }));
+    it("when the session's heartbeat goes stale", () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-stale');
+        yield scratch.db.session.deleteMany({});
+        yield scratch.db.session.create({
+            data: {
+                id: 's-stale',
+                status: 'running',
+                desired_capabilities: '{}',
+                session_capabilities: '{}',
+                node_id: 'n',
+                has_live_video: false,
+                device_udid: 'R5CT',
+                device_platform: 'android',
+                device_version: '14',
+                last_heartbeat_at: new Date(Date.now() - 60 * 60 * 1000),
+            },
+        });
+        yield new OrphanSweeper_1.OrphanSweeper().sweep({ heartbeatIntervalMs: 1000 });
+        expectForgotten('s-stale');
+    }));
+    it("leaves other sessions' elements alone", () => __awaiter(this, void 0, void 0, function* () {
+        seed('s-ends');
+        seed('s-stays');
+        yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).deleteSession(() => __awaiter(this, void 0, void 0, function* () { return null; }), 's-ends');
+        (0, chai_1.expect)(omni.getVirtualElement('omni_ocr_s-stays', 's-stays')).to.not.equal(undefined);
+    }));
+});

package/lib/test/unit/omni-vision-ios-points.spec.js

@@ -92,10 +92,8 @@ describe('Omni-Vision taps in points on iOS', () => {
         (0, chai_1.expect)(tappedAt(ios)).to.deep.equal({ x: 5, y: 11 });
     }));
     it('gives the -custom:ai-text locator an element in points on iOS', () => __awaiter(void 0, void 0, void 0, function* () {
-        var _a;
         const [found] = yield service.findByText(ios, 'OK');
         (0, chai_1.expect)(found.rect).to.deep.equal({ x: 4, y: 10, width: 2, height: 2 });
-        (0, chai_1.expect)((_a = service.getVirtualElement(found.id)) === null || _a === void 0 ? void 0 : _a.rect).to.deep.equal(found.rect);
     }));
     it("doesn't tap on iOS when the window size can't be read", () => __awaiter(void 0, void 0, void 0, function* () {
         ios.getWindowRect.rejects(new Error('WDA is not running'));