@xenon-device-management/xenon 2.13.2 → 2.15.0

package/lib/test/unit/ocr-healing-tier-real-ocr.spec.js

@@ -0,0 +1,113 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sharp_1 = __importDefault(require("sharp"));
+const typedi_1 = require("typedi");
+const OcrHealingProvider_1 = require("../../src/services/healing/OcrHealingProvider");
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const container_registration_1 = require("../helpers/container-registration");
+/**
+ * The OCR healing tier, on a real screenshot read by a real Tesseract worker
+ * (the language data the plugin ships), with nothing stubbed between them.
+ *
+ * Through 2.14 the tier read `result.data.words` from tesseract.js, which
+ * since tesseract.js 6 returns text only unless asked: `words` was undefined,
+ * and the tier healed nothing, ever. Its tests stubbed the OCR with words, so
+ * none of them could see it. ocr-healing-tier.spec.ts covers the matching
+ * rules on stubbed words; this checks the whole tier reads a screen.
+ */
+describe('The OCR healing tier, reading a real screenshot', () => {
+    let restoreContainer;
+    let screenshot;
+    // A phone-like screen: four lines, 200 px apart, each at x 60. "Log out"
+    // and "Check in" come first, so a tier that took a lone "Log" for "Login",
+    // or a lone "in" for "Sign in", answers with the wrong line.
+    const WIDTH = 720;
+    const LINES = { 'Log out': 150, 'Check in': 350, 'Sign in': 550, Login: 750 };
+    const FONT_SIZE = 56;
+    /** Where a line's text is: from its cap height above the baseline to just below it. */
+    const band = (line) => ({
+        top: LINES[line] - FONT_SIZE,
+        bottom: LINES[line] + FONT_SIZE / 4,
+    });
+    before(() => __awaiter(void 0, void 0, void 0, function* () {
+        const text = Object.entries(LINES)
+            .map(([words, y]) => `<text x="60" y="${y}" font-family="Helvetica, Arial, sans-serif" font-size="${FONT_SIZE}" fill="#111">${words}</text>`)
+            .join('');
+        const png = yield (0, sharp_1.default)({
+            create: { width: WIDTH, height: 900, channels: 3, background: '#ffffff' },
+        })
+            .composite([
+            {
+                input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${WIDTH}" height="900">${text}</svg>`),
+            },
+        ])
+            .png()
+            .toBuffer();
+        screenshot = png.toString('base64');
+        // An Omni-Vision of its own, whose worker this suite starts and ends.
+        restoreContainer = (0, container_registration_1.saveRegistrations)(OmniVisionService_1.OmniVisionService);
+        typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+    }));
+    after(() => __awaiter(void 0, void 0, void 0, function* () {
+        const worker = typedi_1.Container.get(OmniVisionService_1.OmniVisionService).sharedWorker;
+        yield (worker === null || worker === void 0 ? void 0 : worker.terminate());
+        restoreContainer();
+    }));
+    /** An Android driver: no element answers an iOS predicate, so the tier says where the text is. */
+    const android = { findElement: () => __awaiter(void 0, void 0, void 0, function* () { return Promise.reject(new Error('unsupported locator')); }) };
+    const heal = (selector) => new OcrHealingProvider_1.OcrHealingProvider().heal({
+        sessionId: 'real-ocr-session',
+        driver: android,
+        strategy: 'xpath',
+        selector,
+        screenshotBase64: screenshot,
+    });
+    /** The healed element's box lies on `line`, starting where its text starts. */
+    const expectOnLine = (rect, line) => {
+        const { top, bottom } = band(line);
+        (0, chai_1.expect)(rect, 'rect').to.be.an('object');
+        (0, chai_1.expect)(rect.y, `top of the box, on "${line}"`).to.be.within(top, bottom);
+        (0, chai_1.expect)(rect.y + rect.height, `bottom of the box, on "${line}"`).to.be.within(top, bottom);
+        (0, chai_1.expect)(rect.x, 'left of the box').to.be.within(40, 80);
+    };
+    it('finds a two-word text on its line, not a lone word of it read earlier', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='Sign in']");
+        (0, chai_1.expect)(healed, 'healed').to.not.equal(null);
+        (0, chai_1.expect)(healed.id).to.match(/^healed_ocr_/);
+        (0, chai_1.expect)(healed.recommendedSelector).to.equal('ocr:text="Sign in"');
+        (0, chai_1.expect)(healed.text).to.equal('Sign in');
+        expectOnLine(healed.rect, 'Sign in');
+        // Both words: wider than "Sign" alone at this size.
+        (0, chai_1.expect)(healed.rect.width).to.be.above(150);
+        (0, chai_1.expect)(healed.confidence).to.be.within(0.6, 1);
+    }));
+    it('does not take a word that is only part of the text', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='Login']");
+        (0, chai_1.expect)(healed, 'healed').to.not.equal(null);
+        (0, chai_1.expect)(healed.text).to.equal('Login');
+        expectOnLine(healed.rect, 'Login');
+    }));
+    it('reads the text from the attribute the selector names, whatever its case', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//android.widget.Button[@content-desc='check IN']");
+        (0, chai_1.expect)(healed, 'healed').to.not.equal(null);
+        (0, chai_1.expect)(healed.text).to.equal('Check in');
+        expectOnLine(healed.rect, 'Check in');
+    }));
+    it('heals nothing when the text is not on the screen', () => __awaiter(void 0, void 0, void 0, function* () {
+        (0, chai_1.expect)(yield heal("//*[@text='Sign up']")).to.equal(null);
+    }));
+});

package/lib/test/unit/ocr-healing-tier.spec.js

@@ -0,0 +1,122 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const OcrHealingProvider_1 = require("../../src/services/healing/OcrHealingProvider");
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const VisualAiHealingProvider_1 = require("../../src/services/healing/VisualAiHealingProvider");
+const AIService_1 = require("../../src/services/AIService");
+/**
+ * Self-healing's OCR tier took the first word OCR read that contained the
+ * text, or was part of it: for `Sign in` that was any lone "in" or "Sign" on
+ * the screen, and for `Login` a "Log". It now matches as Omni-Vision's
+ * `-custom:ai-text` does (ocrTextMatch.ts): the text across neighbouring words
+ * on one line, the first match in reading order.
+ *
+ * OCR is stubbed with the words it would read, in screenshot pixels: no
+ * Tesseract runs. ocr-healing-tier-real-ocr.spec.ts runs the tier on a real
+ * screenshot, which a stub can't stand in for: tesseract.js's own output is
+ * what hid the tier's never healing.
+ */
+describe('The OCR healing tier matches text as Omni-Vision does', () => {
+    const word = (text, x0, y0, x1, y1) => ({
+        text,
+        confidence: 90,
+        bbox: { x0, y0, x1, y1 },
+    });
+    // A heading with "Log" and a lone "in" read first; then "Sign in", far
+    // from "Help" on its line; then "Login".
+    const WORDS = [
+        word('Log', 40, 60, 90, 84),
+        word('in', 300, 60, 320, 84),
+        word('Sign', 100, 200, 150, 224),
+        word('in', 158, 200, 180, 224),
+        word('Help', 600, 200, 650, 224),
+        word('Login', 100, 400, 180, 424),
+    ];
+    let driver;
+    beforeEach(() => {
+        // OCR reads these words: the tier asks Omni-Vision's OCR.
+        sinon_1.default
+            .stub(typedi_1.Container.get(OmniVisionService_1.OmniVisionService), 'performOcr')
+            .resolves({ text: '', words: WORDS });
+        // Android: no element is found by an iOS predicate, so the tier answers
+        // with where the text is.
+        driver = { findElement: sinon_1.default.stub().rejects(new Error('unsupported locator strategy')) };
+    });
+    afterEach(() => sinon_1.default.restore());
+    const heal = (selector) => new OcrHealingProvider_1.OcrHealingProvider().heal({
+        sessionId: 'ocr-session',
+        driver,
+        strategy: 'xpath',
+        selector,
+        screenshotBase64: 'aGVsbG8=',
+    });
+    it('finds a two-word text across its words, not a lone word of it', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='Sign in']");
+        (0, chai_1.expect)(healed, 'healed').to.not.equal(null);
+        (0, chai_1.expect)(healed.recommendedSelector).to.equal('ocr:text="Sign in"');
+        (0, chai_1.expect)(healed.rect).to.deep.equal({ x: 100, y: 200, width: 80, height: 24 });
+    }));
+    it('hands on the text it read, for getText on the element it returns', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='sign IN']");
+        (0, chai_1.expect)(healed.text).to.equal('Sign in');
+        (0, chai_1.expect)(healed.message).to.not.equal(healed.text);
+    }));
+    it('does not take a word that is only part of the text', () => __awaiter(void 0, void 0, void 0, function* () {
+        const healed = yield heal("//*[@text='Login']");
+        (0, chai_1.expect)(healed, 'healed').to.not.equal(null);
+        (0, chai_1.expect)(healed.recommendedSelector).to.equal('ocr:text="Login"');
+        (0, chai_1.expect)(healed.rect).to.deep.equal({ x: 100, y: 400, width: 80, height: 24 });
+    }));
+    it('finds nothing when the text is not on the screen', () => __awaiter(void 0, void 0, void 0, function* () {
+        (0, chai_1.expect)(yield heal("//*[@text='Sign up']")).to.equal(null);
+    }));
+});
+/**
+ * A healed element's id names it in the server's memory of virtual elements,
+ * which every session shares. `healed_ocr_<ms>` was the same for two heals in
+ * one millisecond, so the second overwrote the first.
+ */
+describe('Elements healed by OCR or Visual AI', () => {
+    afterEach(() => sinon_1.default.restore());
+    const context = {
+        sessionId: 'ids-session',
+        driver: { findElement: () => __awaiter(void 0, void 0, void 0, function* () { return Promise.reject(new Error('no predicate')); }) },
+        strategy: 'xpath',
+        selector: "//*[@text='Login']",
+        screenshotBase64: 'aGVsbG8=',
+    };
+    it('get ids of their own when healed in the same millisecond', () => __awaiter(void 0, void 0, void 0, function* () {
+        var _a, _b, _c, _d;
+        sinon_1.default.stub(Date, 'now').returns(1700000000000);
+        sinon_1.default.stub(typedi_1.Container.get(OmniVisionService_1.OmniVisionService), 'performOcr').resolves({
+            text: '',
+            words: [{ text: 'Login', confidence: 90, bbox: { x0: 1, y0: 1, x1: 50, y1: 20 } }],
+        });
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'isEnabled').returns(true);
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'visualFind').resolves({ x: 100, y: 100 });
+        const ids = [
+            (_a = (yield new OcrHealingProvider_1.OcrHealingProvider().heal(context))) === null || _a === void 0 ? void 0 : _a.id,
+            (_b = (yield new OcrHealingProvider_1.OcrHealingProvider().heal(context))) === null || _b === void 0 ? void 0 : _b.id,
+            (_c = (yield new VisualAiHealingProvider_1.VisualAiHealingProvider().heal(context))) === null || _c === void 0 ? void 0 : _c.id,
+            (_d = (yield new VisualAiHealingProvider_1.VisualAiHealingProvider().heal(context))) === null || _d === void 0 ? void 0 : _d.id,
+        ];
+        (0, chai_1.expect)(ids.every(Boolean), String(ids)).to.equal(true);
+        (0, chai_1.expect)(new Set(ids).size, String(ids)).to.equal(4);
+    }));
+});

package/lib/test/unit/ocr-offline.spec.js

@@ -0,0 +1,213 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = __importDefault(require("fs"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const sharp_1 = __importDefault(require("sharp"));
+const sinon_1 = __importDefault(require("sinon"));
+const tesseract_js_1 = __importDefault(require("tesseract.js"));
+const typedi_1 = require("typedi");
+const package_json_1 = __importDefault(require("../../package.json"));
+const ocrData_1 = require("../../src/services/ocr/ocrData");
+const OcrHealingProvider_1 = require("../../src/services/healing/OcrHealingProvider");
+const OmniVisionService_1 = require("../../src/services/omni-vision/OmniVisionService");
+const container_registration_1 = require("../helpers/container-registration");
+/**
+ * tesseract.js downloads its English data (eng.traineddata) from
+ * cdn.jsdelivr.net the first time a worker starts, and caches it in the
+ * server's working directory. A server with no internet couldn't run OCR at
+ * all: the first OCR call never finished, and every later one waited for it.
+ * One with internet left a 5 MB eng.traineddata wherever it was started (the
+ * home directory, ~/.appium).
+ *
+ * The data now ships with the plugin, read from there and cached nowhere.
+ */
+describe('OCR without the internet', () => {
+    // The upstream file, @tesseract.js-data/eng@1.0.0 4.0.0_best_int/eng.traineddata.gz:
+    // what tesseract.js 7 downloads by default (LSTM only).
+    const UPSTREAM_SHA256 = '45b4cb346724ac1774f1c36f42f182b887bcdb28ebe63e6fff90ac41f3fcff91';
+    const tmpDirs = [];
+    const tmpDir = () => {
+        const dir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), 'xenon-ocr-'));
+        tmpDirs.push(dir);
+        return dir;
+    };
+    afterEach(() => {
+        sinon_1.default.restore();
+        for (const dir of tmpDirs.splice(0))
+            fs_1.default.rmSync(dir, { recursive: true, force: true });
+    });
+    /** A white image with `text` on it, as a phone's screenshot would have. */
+    const imageOf = (text) => (0, sharp_1.default)({ create: { width: 900, height: 220, channels: 3, background: '#ffffff' } })
+        .composite([
+        {
+            input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="900" height="220">` +
+                `<text x="40" y="140" font-family="Helvetica, Arial, sans-serif" font-size="96" fill="#000">${text}</text>` +
+                `</svg>`),
+        },
+    ])
+        .png()
+        .toBuffer();
+    describe('the English data', () => {
+        it('ships with the plugin, byte for byte the file tesseract.js would download', () => {
+            (0, chai_1.expect)(ocrData_1.OCR_DATA_FILE).to.equal(path_1.default.join(ocrData_1.OCR_DATA_DIR, 'eng.traineddata.gz'));
+            const sha = crypto_1.default.createHash('sha256').update(fs_1.default.readFileSync(ocrData_1.OCR_DATA_FILE)).digest('hex');
+            (0, chai_1.expect)(sha).to.equal(UPSTREAM_SHA256);
+            (0, chai_1.expect)(ocrData_1.OCR_DATA_SHA256).to.equal(UPSTREAM_SHA256);
+            const readme = fs_1.default.readFileSync(path_1.default.join(ocrData_1.OCR_DATA_DIR, 'README.md'), 'utf8');
+            (0, chai_1.expect)(readme).to.include(UPSTREAM_SHA256);
+        });
+        it('is copied into lib/ by the build, which is what the npm package ships', () => {
+            const copy = package_json_1.default.scripts['build:copy'];
+            (0, chai_1.expect)(copy).to.include('mkdir -p lib/src/services/ocr/vendor');
+            (0, chai_1.expect)(copy).to.include('cp src/services/ocr/vendor/*.gz');
+            (0, chai_1.expect)(package_json_1.default.files).to.include('lib');
+        });
+        it('is read from there, never fetched, and cached nowhere', () => {
+            const options = (0, ocrData_1.ocrWorkerOptions)();
+            (0, chai_1.expect)(options.langPath).to.equal(ocrData_1.OCR_DATA_DIR);
+            (0, chai_1.expect)(String(options.langPath)).not.to.match(/^[a-z]+:\/\//i);
+            (0, chai_1.expect)(options.cacheMethod).to.equal('none');
+            (0, chai_1.expect)(options.gzip).to.equal(true);
+        });
+    });
+    describe('a real OCR worker', () => {
+        let cwd;
+        beforeEach(() => {
+            cwd = process.cwd();
+        });
+        afterEach(() => {
+            process.chdir(cwd);
+        });
+        it('reads text, and writes nothing into the working directory', () => __awaiter(void 0, void 0, void 0, function* () {
+            const workDir = tmpDir();
+            process.chdir(workDir);
+            const worker = yield (0, ocrData_1.createOcrWorker)();
+            try {
+                const { data } = yield worker.recognize(yield imageOf('Sign in'));
+                (0, chai_1.expect)(data.text).to.match(/sign\s*in/i);
+            }
+            finally {
+                yield worker.terminate();
+            }
+            (0, chai_1.expect)(fs_1.default.readdirSync(workDir)).to.deep.equal([]);
+        }));
+        it('fails with a reason, rather than wait for ever, when the data is missing', () => __awaiter(void 0, void 0, void 0, function* () {
+            const started = Date.now();
+            const failure = yield (0, ocrData_1.createOcrWorker)(tmpDir()).then(() => chai_1.expect.fail('a worker with no data should not start'), (err) => err);
+            (0, chai_1.expect)(failure.message).to.include('eng.traineddata.gz');
+            (0, chai_1.expect)(Date.now() - started).to.be.below(5000);
+        }));
+        // Given data it can't load, tesseract.js 7 throws from its own message
+        // handler, and the server exits on an uncaught exception: the data is
+        // checked before a worker sees it.
+        for (const [damage, bytes] of [
+            ['not tesseract data at all', () => Buffer.from('not tesseract data')],
+            ['cut short', () => fs_1.default.readFileSync(ocrData_1.OCR_DATA_FILE).subarray(0, 100000)],
+        ]) {
+            it(`fails with a reason, before any worker starts, when the data is ${damage}`, () => __awaiter(void 0, void 0, void 0, function* () {
+                const dir = tmpDir();
+                fs_1.default.writeFileSync(path_1.default.join(dir, 'eng.traineddata.gz'), bytes());
+                const createWorker = sinon_1.default.spy(tesseract_js_1.default, 'createWorker');
+                const failure = yield (0, ocrData_1.createOcrWorker)(dir).then(() => chai_1.expect.fail('a worker with damaged data should not start'), (err) => err);
+                (0, chai_1.expect)(failure.message).to.include('damaged');
+                (0, chai_1.expect)(createWorker.called).to.equal(false);
+            }));
+        }
+    });
+    describe('a worker that will not start', () => {
+        /** A directory holding a good copy of the data, so this test's memory is its own. */
+        const dataDir = () => {
+            const dir = tmpDir();
+            fs_1.default.copyFileSync(ocrData_1.OCR_DATA_FILE, path_1.default.join(dir, 'eng.traineddata.gz'));
+            return dir;
+        };
+        it('is remembered for a minute: OCR fails at once rather than start another', () => __awaiter(void 0, void 0, void 0, function* () {
+            // tesseract.js hands back no worker when its start fails, so its thread
+            // can't be ended; starting one per OCR call would leave one per call.
+            const clock = sinon_1.default.useFakeTimers({ now: Date.now(), toFake: ['Date'] });
+            const createWorker = sinon_1.default
+                .stub(tesseract_js_1.default, 'createWorker')
+                .callsFake((_langs, _oem, options) => {
+                setTimeout(() => options.errorHandler('the OCR engine failed to load'), 5);
+                return new Promise(() => undefined);
+            });
+            const dir = dataDir();
+            const reason = (p) => p.then(() => chai_1.expect.fail('the worker should not start'), (err) => err.message);
+            (0, chai_1.expect)(yield reason((0, ocrData_1.createOcrWorker)(dir))).to.include('the OCR engine failed to load');
+            (0, chai_1.expect)(yield reason((0, ocrData_1.createOcrWorker)(dir))).to.include('the OCR engine failed to load');
+            (0, chai_1.expect)(createWorker.callCount).to.equal(1);
+            clock.tick(60001);
+            yield reason((0, ocrData_1.createOcrWorker)(dir));
+            (0, chai_1.expect)(createWorker.callCount).to.equal(2);
+        }));
+    });
+    describe('every place that runs OCR', () => {
+        let restoreContainer;
+        beforeEach(() => {
+            restoreContainer = (0, container_registration_1.saveRegistrations)(OmniVisionService_1.OmniVisionService);
+        });
+        afterEach(() => {
+            restoreContainer();
+        });
+        it("the OCR healing tier reads the shipped data, with Omni-Vision's worker", () => __awaiter(void 0, void 0, void 0, function* () {
+            // The tier reads the screenshot through Omni-Vision (findTextInScreenshot),
+            // so it shares that worker and leaves it running for the next read.
+            const worker = {
+                setParameters: sinon_1.default.stub().resolves(),
+                recognize: sinon_1.default.stub().resolves({ data: { text: '', words: [], hocr: '', tsv: '' } }),
+                terminate: sinon_1.default.stub().resolves(),
+            };
+            const createWorker = sinon_1.default.stub(tesseract_js_1.default, 'createWorker').resolves(worker);
+            typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+            yield new OcrHealingProvider_1.OcrHealingProvider().heal({
+                screenshotBase64: (yield imageOf('Login')).toString('base64'),
+                selector: "//*[@text='Login']",
+                strategy: 'xpath',
+                driver: {},
+            });
+            (0, chai_1.expect)(createWorker.calledOnce).to.equal(true);
+            (0, chai_1.expect)(createWorker.firstCall.args[0]).to.equal('eng');
+            (0, chai_1.expect)(createWorker.firstCall.args[2]).to.include({
+                langPath: ocrData_1.OCR_DATA_DIR,
+                cacheMethod: 'none',
+            });
+            (0, chai_1.expect)(worker.recognize.calledOnce).to.equal(true);
+            (0, chai_1.expect)(worker.terminate.called).to.equal(false);
+        }));
+        it("Omni-Vision's worker reads the shipped data", () => __awaiter(void 0, void 0, void 0, function* () {
+            const worker = {
+                setParameters: sinon_1.default.stub().resolves(),
+                recognize: sinon_1.default.stub().resolves({ data: { text: '', words: [], hocr: '', tsv: '' } }),
+                terminate: sinon_1.default.stub().resolves(),
+            };
+            const createWorker = sinon_1.default.stub(tesseract_js_1.default, 'createWorker').resolves(worker);
+            typedi_1.Container.set(OmniVisionService_1.OmniVisionService, new OmniVisionService_1.OmniVisionService());
+            const driver = {
+                getScreenshot: sinon_1.default.stub().resolves((yield imageOf('Login')).toString('base64')),
+            };
+            yield typedi_1.Container.get(OmniVisionService_1.OmniVisionService).findByText(driver, 'Login');
+            (0, chai_1.expect)(createWorker.calledOnce).to.equal(true);
+            (0, chai_1.expect)(createWorker.firstCall.args[0]).to.equal('eng');
+            (0, chai_1.expect)(createWorker.firstCall.args[2]).to.include({
+                langPath: ocrData_1.OCR_DATA_DIR,
+                cacheMethod: 'none',
+            });
+        }));
+    });
+});

package/lib/test/unit/ocr-text-match.spec.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const chai_1 = require("chai");
+const ocrTextMatch_1 = require("../../src/services/omni-vision/ocrTextMatch");
+describe('ocrTextMatch', () => {
+    const word = (text, x0, y0, x1, y1, confidence = 90) => ({ text, x0, y0, x1, y1, confidence });
+    it('groups words into lines and runs, in reading order, despite a slightly uneven baseline', () => {
+        const runs = (0, ocrTextMatch_1.wordRuns)([
+            word('Done', 900, 40, 980, 70),
+            word('in', 158, 202, 180, 226),
+            word('Cancel', 20, 42, 120, 70),
+            word('Sign', 100, 200, 150, 224),
+        ]);
+        (0, chai_1.expect)(runs.map((r) => r.map((w) => w.text))).to.deep.equal([
+            ['Cancel'],
+            ['Done'],
+            ['Sign', 'in'],
+        ]);
+    });
+    it('finds every occurrence, once per set of words, with the lowest confidence of the words', () => {
+        const words = [
+            word('Add', 10, 10, 50, 30, 95),
+            word('to', 56, 10, 70, 30, 70),
+            word('cart', 76, 10, 120, 30, 88),
+            word('Add', 10, 100, 50, 120),
+            word('to', 56, 100, 70, 120),
+            word('cart', 76, 100, 120, 120),
+        ];
+        const found = (0, ocrTextMatch_1.findText)(words, '  add   TO cart ');
+        (0, chai_1.expect)(found.map((m) => [m.text, m.y0, m.confidence])).to.deep.equal([
+            ['Add to cart', 10, 70],
+            ['Add to cart', 100, 90],
+        ]);
+        (0, chai_1.expect)(found[0]).to.include({ x0: 10, x1: 120, y1: 30 });
+    });
+    it('matches a one-word text inside a word once, however often it appears there', () => {
+        (0, chai_1.expect)((0, ocrTextMatch_1.findText)([word('banana', 0, 0, 60, 20)], 'an').map((m) => m.text)).to.deep.equal([
+            'banana',
+        ]);
+    });
+    it('matches nothing for an empty text', () => {
+        (0, chai_1.expect)((0, ocrTextMatch_1.findText)([word('Sign', 0, 0, 40, 20)], '   ')).to.deep.equal([]);
+    });
+});