@xenon-device-management/xenon 2.13.2 → 2.15.0

package/lib/src/services/omni-vision/OmniVisionService.js

@@ -1,43 +1,10 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
 var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
     var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
     if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -52,20 +19,36 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 var OmniVisionService_1;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.OmniVisionService = void 0;
+exports.OmniVisionService = exports.MAX_VIRTUAL_ELEMENTS = exports.MAX_VIRTUAL_ELEMENTS_PER_SESSION = void 0;
 const typedi_1 = require("typedi");
-const tesseract_js_1 = __importStar(require("tesseract.js"));
+const tesseract_js_1 = require("tesseract.js");
+const ocrData_1 = require("../ocr/ocrData");
 const sharp_1 = __importDefault(require("sharp"));
 const AIService_1 = require("../AIService");
 const logger_1 = __importDefault(require("../../logger"));
 const crypto_1 = __importDefault(require("crypto"));
+const ocrTextMatch_1 = require("./ocrTextMatch");
+const screenScale_1 = require("./screenScale");
+/**
+ * How many virtual elements Xenon keeps, for one session and in all. The
+ * oldest go first: a test acts on what it found last, and a command on an
+ * element that went answers "no such element".
+ */
+exports.MAX_VIRTUAL_ELEMENTS_PER_SESSION = 1000;
+exports.MAX_VIRTUAL_ELEMENTS = 10000;
 // Mobile UIs have sparse text on complex backgrounds - reduce contrast to preserve readability
 const DEFAULT_CONTRAST = 1.2; // Slightly above 1.0 for mild enhancement (was 0.8 which was too aggressive)
 const UPSCALE_THRESHOLD = 800; // Only upscale very small images (was 1500)
 let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
     constructor() {
         this.logger = logger_1.default.scope('OmniVision');
-        this.virtualElementStore = new Map();
+        /**
+         * The virtual elements tests may still act on, by id, each with the session
+         * that found it, oldest first. Through 2.14 every element ever found stayed
+         * here until the server restarted, and any session could act on another's.
+         */
+        this.elements = new Map();
+        this.bySession = new Map();
         this.sharedWorker = null;
         this.workerBusy = false;
         this.uiLensCache = new Map();
@@ -75,7 +58,7 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
             if (this.sharedWorker)
                 return this.sharedWorker;
             this.logger.info('Initializing persistent OCR worker with explicit configuration...');
-            const worker = yield tesseract_js_1.default.createWorker('eng');
+            const worker = yield (0, ocrData_1.createOcrWorker)();
             // PSM.SPARSE_TEXT is optimized for irregular text layouts like mobile UIs
             // AUTO mode assumes document structure which fails on sparse mobile screens
             yield worker.setParameters({
@@ -379,10 +362,32 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
             }
         });
     }
+    /** The OCR words with usable boxes, in the screenshot's pixels. */
+    wordBoxes(words) {
+        return words.map((w) => this.normalizeWordBBox(w)).filter(Boolean);
+    }
+    /** Where `text` is among OCR's words, read with over 60% confidence, in reading order. */
+    textMatches(words, text) {
+        return (0, ocrTextMatch_1.findText)(this.wordBoxes(words), String(text !== null && text !== void 0 ? text : '')).filter((m) => m.confidence > 60);
+    }
     /**
-     * Proactive OCR Search: Finds elements matching text even if not in XML
+     * Where `text` is on a screenshot, matched as `-custom:ai-text` matches it:
+     * in the screenshot's pixels, in reading order. Self-healing's OCR tier
+     * uses it. Throws when OCR fails.
      */
+    findTextInScreenshot(screenshotBase64, text) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { words } = yield this.performOcr(Buffer.from(screenshotBase64, 'base64'));
+            return this.textMatches(words, text);
+        });
+    }
     /**
+     * Proactive OCR Search: Finds elements matching text even if not in XML.
+     *
+     * The text may be several words ("Sign in"); see ocrTextMatch.ts. Each
+     * element's rect is in the driver's coordinates (points on iOS), like a
+     * real element's, so tapping it lands on the text.
+     *
      * `throwOnError`: fail instead of answering "no match" when the screenshot
      * or OCR fails. Device control's "Test locator" asks for it; an Appium
      * findElement doesn't, and turns the empty list into NoSuchElement.
@@ -398,22 +403,16 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
                     this.logger.warn(`OCR proactive search yielded no words. Text content: "${ocrText.trim().substring(0, 100)}..."`);
                     return [];
                 }
-                const matches = words.filter((w) => w.text && w.text.toLowerCase().includes(text.toLowerCase()) && w.confidence > 60);
-                return matches.map((m) => {
-                    const el = {
-                        id: `omni_ocr_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`,
-                        text: m.text,
-                        rect: {
-                            x: m.bbox.x0,
-                            y: m.bbox.y0,
-                            width: m.bbox.x1 - m.bbox.x0,
-                            height: m.bbox.y1 - m.bbox.y0,
-                        },
-                        confidence: m.confidence / 100,
-                    };
-                    this.virtualElementStore.set(el.id, el);
-                    return el;
-                });
+                const matches = this.textMatches(words, text);
+                if (matches.length === 0)
+                    return [];
+                const scale = yield (0, screenScale_1.screenScaleOf)(driver, screenshot);
+                return matches.map((m) => ({
+                    id: `omni_ocr_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`,
+                    text: m.text,
+                    rect: (0, screenScale_1.toDriverRect)({ x: m.x0, y: m.y0, width: m.x1 - m.x0, height: m.y1 - m.y0 }, scale),
+                    confidence: m.confidence / 100,
+                }));
             }
             catch (err) {
                 this.logger.error(`OCR proactive search failed: ${err.message}`);
@@ -433,18 +432,14 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
                 const screenshot = yield driver.getScreenshot();
                 const coordinates = yield AIService_1.AI_SERVICE.visualFind(screenshot, iconDescription, opts);
                 if (coordinates) {
-                    const el = {
-                        id: `omni_ai_${Date.now()}`,
-                        rect: {
-                            x: coordinates.x - 20,
-                            y: coordinates.y - 20,
-                            width: 40,
-                            height: 40,
-                        },
+                    // The AI answers in the screenshot's pixels; the rect is the driver's
+                    // (points on iOS), like a real element's.
+                    const scale = yield (0, screenScale_1.screenScaleOf)(driver, screenshot);
+                    return {
+                        id: `omni_ai_${Date.now()}_${Math.random().toString(36).slice(2, 7)}`,
+                        rect: (0, screenScale_1.toDriverRect)({ x: coordinates.x - 20, y: coordinates.y - 20, width: 40, height: 40 }, scale),
                         confidence: 0.85,
                     };
-                    this.virtualElementStore.set(el.id, el);
-                    return el;
                 }
             }
             catch (err) {
@@ -460,6 +455,7 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
      */
     analyzeScreen(driver) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             this.logger.info('Performing full screen analysis...');
             try {
                 const screenshot = yield driver.getScreenshot();
@@ -470,32 +466,28 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
                 this.validateBuffer(buffer, 'Screen Analysis');
                 // 1. Get all text via OCR using the robust worker flow
                 const { text: ocrText, words } = yield this.performOcr(buffer);
-                // 2. Ask AI for qualitative analysis (Non-blocking)
-                let aiAnalysis = null;
+                // 2. Ask the AI provider about the screen, with the screenshot. Its
+                // failure leaves the OCR answer standing, and says why there are no
+                // insights. Through 2.13.2 this sent the screenshot's base64 as a file
+                // path, so the provider got no image and a "why did this test fail"
+                // prompt, and answered about no screen at all.
+                let aiInsights = null;
+                let aiError;
                 try {
-                    aiAnalysis = yield AIService_1.AI_SERVICE.analyzeFailure({
-                        sessionId: driver.sessionId,
-                        failureReason: 'Screen Analysis Request',
-                        commandLogs: [],
-                        deviceLogs: [],
-                        screenshotPath: screenshot, // In a real scenario, we'd save this to a file first or pass base64
-                    });
+                    aiInsights = yield AIService_1.AI_SERVICE.describeScreen(screenshot);
                 }
                 catch (aiErr) {
-                    this.logger.warn(`AI insights skipped: ${aiErr.message}`);
+                    aiError = (_a = aiErr === null || aiErr === void 0 ? void 0 : aiErr.message) !== null && _a !== void 0 ? _a : String(aiErr);
+                    this.logger.warn(`AI insights skipped: ${aiError}`);
                 }
-                return {
-                    timestamp: new Date().toISOString(),
-                    ocr: {
+                return Object.assign({ timestamp: new Date().toISOString(), ocr: {
                         text: ocrText,
                         words: words.map((w) => ({
                             text: w.text || '',
                             confidence: w.confidence || 0,
                             bbox: w.bbox || { x0: 0, y0: 0, x1: 0, y1: 0 },
                         })),
-                    },
-                    ai_insights: aiAnalysis,
-                };
+                    }, ai_insights: aiInsights }, (aiError !== undefined ? { ai_insights_error: aiError } : {}));
             }
             catch (err) {
                 this.logger.error(`Screen analysis failed: ${err.message}`);
@@ -503,14 +495,56 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
             }
         });
     }
-    addVirtualElement(element) {
-        this.virtualElementStore.set(element.id, element);
+    /**
+     * Keep `element` for `sessionId`, whose test was handed its id (a
+     * `-custom:ai-*` find, a heal), until the session ends (`forgetSession`)
+     * or the limits push it out. A find doesn't keep what it finds: device
+     * control's locator test only shows the ids, and a tap uses its element at
+     * once.
+     */
+    remember(sessionId, element) {
+        this.drop(element.id);
+        this.elements.set(element.id, { element, sessionId });
+        let ids = this.bySession.get(sessionId);
+        if (!ids) {
+            ids = new Set();
+            this.bySession.set(sessionId, ids);
+        }
+        ids.add(element.id);
+        if (ids.size > exports.MAX_VIRTUAL_ELEMENTS_PER_SESSION)
+            this.drop(ids.values().next().value);
+        if (this.elements.size > exports.MAX_VIRTUAL_ELEMENTS)
+            this.drop(this.elements.keys().next().value);
     }
-    getVirtualElement(id) {
-        return this.virtualElementStore.get(id);
+    /** The element, when `sessionId` found it and still has it. */
+    getVirtualElement(id, sessionId) {
+        const held = this.elements.get(id);
+        return held && held.sessionId === sessionId ? held.element : undefined;
+    }
+    /** Drop every element a session found: it has ended (sessionMemory.ts). */
+    forgetSession(sessionId) {
+        var _a;
+        for (const id of (_a = this.bySession.get(sessionId)) !== null && _a !== void 0 ? _a : [])
+            this.elements.delete(id);
+        this.bySession.delete(sessionId);
+    }
+    drop(id) {
+        const held = this.elements.get(id);
+        if (!held)
+            return;
+        this.elements.delete(id);
+        const ids = this.bySession.get(held.sessionId);
+        ids === null || ids === void 0 ? void 0 : ids.delete(id);
+        if (ids && ids.size === 0)
+            this.bySession.delete(held.sessionId);
     }
     /**
      * Omni-Click: Find a target by OCR text and click its center using W3C actions.
+     *
+     * The text may be several words ("Sign in"); see ocrTextMatch.ts. The tap
+     * and `target` are in the driver's coordinates (points on iOS). A failed
+     * screenshot or OCR, or a screen size that can't be read on iOS, fails;
+     * `clicked: false` means the text was looked for and not found.
      */
     omniClickByText(driver, req) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -533,22 +567,21 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
             const screenshot = yield driver.getScreenshot();
             const buffer = Buffer.from(screenshot, 'base64');
             const { words } = yield this.performOcr(buffer);
-            const normalized = words.map((w) => this.normalizeWordBBox(w)).filter(Boolean);
-            const matches = normalized
-                .filter((w) => w.text.toLowerCase().includes(text.toLowerCase()))
-                .sort((a, b) => (b.confidence || 0) - (a.confidence || 0) || a.y0 - b.y0 || a.x0 - b.x0);
+            const matches = (0, ocrTextMatch_1.findText)(this.wordBoxes(words), text).sort((a, b) => (b.confidence || 0) - (a.confidence || 0) || a.y0 - b.y0 || a.x0 - b.x0);
             if (matches.length === 0) {
                 return { clicked: false, message: `No OCR match found for text "${text}"` };
             }
             const pick = matches[Math.min(index - 1, matches.length - 1)];
+            const scale = yield (0, screenScale_1.screenScaleOf)(driver, screenshot);
+            const found = (0, screenScale_1.toDriverRect)({ x: pick.x0, y: pick.y0, width: pick.x1 - pick.x0, height: pick.y1 - pick.y0 }, scale);
             const rect = {
-                x: Math.max(0, Math.round(pick.x0)),
-                y: Math.max(0, Math.round(pick.y0)),
-                width: Math.max(1, Math.round(pick.x1 - pick.x0)),
-                height: Math.max(1, Math.round(pick.y1 - pick.y0)),
+                x: Math.max(0, Math.round(found.x)),
+                y: Math.max(0, Math.round(found.y)),
+                width: Math.max(1, Math.round(found.width)),
+                height: Math.max(1, Math.round(found.height)),
             };
-            const cx = Math.round(rect.x + rect.width / 2);
-            const cy = Math.round(rect.y + rect.height / 2);
+            const cx = Math.round(found.x + found.width / 2);
+            const cy = Math.round(found.y + found.height / 2);
             // Click via touch actions
             yield driver.performActions([
                 {
@@ -583,6 +616,10 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
     }
     /**
      * Omni-Click by Icon: Find a target by visual description and click its center.
+     *
+     * Fails when it can't look (no AI provider, a failed screenshot or AI call);
+     * `clicked: false` means the AI looked and found nothing. Through 2.13.2 every
+     * one of those answered `clicked: false`, "No visual match found".
      */
     omniClickByIcon(driver, req) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -596,7 +633,7 @@ let OmniVisionService = OmniVisionService_1 = class OmniVisionService {
                 };
             }
             this.logger.info(`Omni-Click: searching for icon "${icon}" via AI...`);
-            const element = yield this.findByIcon(driver, icon);
+            const element = yield this.findByIcon(driver, icon, { throwOnError: true });
             if (!element) {
                 return { clicked: false, message: `No visual match found for icon "${icon}"` };
             }

package/lib/src/services/omni-vision/VisionAssertionService.js

@@ -20,6 +20,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VisionAssertionService = void 0;
 const typedi_1 = require("typedi");
+const base_driver_1 = require("@appium/base-driver");
 const AIService_1 = require("../AIService");
 const logger_1 = __importDefault(require("../../logger"));
 let VisionAssertionService = class VisionAssertionService {
@@ -27,38 +28,40 @@ let VisionAssertionService = class VisionAssertionService {
         this.logger = logger_1.default.scope('VisionAssertion');
     }
     /**
-     * Asserts a visual state using AI reasoning
+     * `xenon: assertVisualState`: whether `instruction`, a condition in plain
+     * words, holds on the device's screen now, as the AI provider judges it from
+     * a screenshot.
+     *
+     * Answers `{ result, message }` only with the provider's own verdict.
+     * Whenever there is none it fails, saying the condition was not checked: no
+     * condition, no screenshot, no AI provider, a failed or rate-limited call,
+     * or an answer that isn't true or false. It never answers `false` for "could
+     * not check": a test asserting that something is absent would pass. Through
+     * 2.13.2 it answered `{ result: true, message: 'Assertion placeholder' }` in
+     * all of those cases.
      */
     assertState(driver, instruction) {
         return __awaiter(this, void 0, void 0, function* () {
-            this.logger.info(`Asserting visual state: "${instruction}"`);
+            var _a;
+            const condition = typeof instruction === 'string' ? instruction.trim() : '';
+            if (!condition) {
+                throw new base_driver_1.errors.InvalidArgumentError('assertVisualState needs a condition to check, as a string or { instruction }, ' +
+                    "for example 'The cart is empty'.");
+            }
+            this.logger.info(`Asserting visual state: "${condition}"`);
+            let screenshot;
             try {
-                const screenshot = yield driver.getScreenshot();
-                const prompt = `
-        You are a visual testing quality agent. 
-        Your task is to verify if the following condition is true based on the provided screenshot.
-        
-        Condition: "${instruction}"
-        
-        Response Format: JSON only, strictly { "result": boolean, "message": "detailed explanation of what you see" }.
-      `;
-                const response = yield AIService_1.AI_SERVICE.visualFind(screenshot, prompt);
-                // Note: We use visualFind which calls analyze. We expect a JSON response.
-                // Since analyze returns raw text, we attempt to parse it.
-                if (response && response.result !== undefined) {
-                    return {
-                        result: response.result,
-                        message: response.message || 'Assertion evaluated successfully.',
-                    };
-                }
-                // Fallback: If AI_SERVICE.visualFind was used literally, it might not return what we expect.
-                // Let's assume a more direct path if needed, but for now we'll stick to this pattern.
-                return { result: true, message: 'Assertion placeholder' };
+                screenshot = yield driver.getScreenshot();
             }
             catch (err) {
-                this.logger.error(`Visual assertion failed: ${err.message}`);
-                return { result: false, message: `Error: ${err.message}` };
+                throw new Error(`No screenshot could be taken, so the condition was not checked: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+            }
+            if (!screenshot) {
+                throw new Error('No screenshot could be taken, so the condition was not checked.');
             }
+            const verdict = yield AIService_1.AI_SERVICE.assertVisual(screenshot, condition);
+            this.logger.info(`"${condition}": ${verdict.result} (${verdict.message})`);
+            return verdict;
         });
     }
 };

package/lib/src/services/omni-vision/ocrTextMatch.js

@@ -0,0 +1,105 @@
+"use strict";
+/**
+ * Finding a text, one word or several, among the words OCR read off a
+ * screenshot. Tesseract gives one box per word, so a text with a space in it
+ * ("Sign in") is looked for across neighbouring words on one line, and its
+ * box is the box around the words it covers.
+ *
+ * Pure: boxes in, boxes out, in the screenshot's pixels.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wordRuns = wordRuns;
+exports.findText = findText;
+/**
+ * Two words further apart than this many times their height are not
+ * neighbours: a space is about a third of a word's height, and a gap this
+ * wide is another column, such as "Cancel" and "Done" at either end of a bar.
+ */
+const MAX_GAP_IN_HEIGHTS = 1.5;
+const normalize = (text) => text.toLowerCase().replace(/\s+/g, ' ').trim();
+const heightOf = (w) => Math.max(1, w.y1 - w.y0);
+/** Whether a word sits on a line: their heights overlap by at least half the shorter one. */
+function onLine(line, word) {
+    const overlap = Math.min(line.y1, word.y1) - Math.max(line.y0, word.y0);
+    return overlap > 0 && overlap >= Math.min(heightOf(line), heightOf(word)) / 2;
+}
+/**
+ * The words in runs of neighbours: on one line, left to right, with no wide
+ * gap between them. Runs are in reading order.
+ */
+function wordRuns(words) {
+    const lines = [];
+    for (const word of [...words].sort((a, b) => a.y0 - b.y0 || a.x0 - b.x0)) {
+        const line = lines.find((l) => onLine(l, word));
+        if (line) {
+            line.words.push(word);
+            line.y0 = Math.min(line.y0, word.y0);
+            line.y1 = Math.max(line.y1, word.y1);
+        }
+        else {
+            lines.push({ y0: word.y0, y1: word.y1, words: [word] });
+        }
+    }
+    const runs = [];
+    for (const line of lines.sort((a, b) => a.y0 - b.y0)) {
+        let run = [];
+        for (const word of line.words.sort((a, b) => a.x0 - b.x0)) {
+            const previous = run[run.length - 1];
+            const gap = previous ? word.x0 - previous.x1 : 0;
+            if (previous && gap > MAX_GAP_IN_HEIGHTS * Math.max(heightOf(previous), heightOf(word))) {
+                runs.push(run);
+                run = [];
+            }
+            run.push(word);
+        }
+        if (run.length)
+            runs.push(run);
+    }
+    return runs;
+}
+/**
+ * Every place `text` appears, ignoring case and runs of spaces, within a word
+ * or across neighbouring words: "Sign in" in "Sign" "in", "password" in
+ * "password?". Each match covers the words it touches: its box surrounds
+ * them, its text is theirs joined with spaces, and its confidence is the
+ * lowest of theirs. A one-word text matches inside one word, as it always
+ * did. Matches are in reading order.
+ */
+function findText(words, text) {
+    const wanted = normalize(text);
+    if (!wanted)
+        return [];
+    const matches = [];
+    for (const run of wordRuns(words)) {
+        // The run as one string, remembering where each word sits in it.
+        let line = '';
+        const spans = [];
+        for (const word of run) {
+            if (line)
+                line += ' ';
+            const start = line.length;
+            line += normalize(word.text);
+            spans.push({ start, end: line.length });
+        }
+        const seen = new Set();
+        for (let at = line.indexOf(wanted); at !== -1; at = line.indexOf(wanted, at + 1)) {
+            const end = at + wanted.length;
+            const first = spans.findIndex((s) => s.end > at);
+            const last = spans.findIndex((s) => s.end >= end);
+            const key = `${first}-${last}`;
+            if (first === -1 || last === -1 || seen.has(key))
+                continue;
+            seen.add(key);
+            const covered = run.slice(first, last + 1);
+            matches.push({
+                x0: Math.min(...covered.map((w) => w.x0)),
+                y0: Math.min(...covered.map((w) => w.y0)),
+                x1: Math.max(...covered.map((w) => w.x1)),
+                y1: Math.max(...covered.map((w) => w.y1)),
+                text: covered.map((w) => w.text).join(' '),
+                confidence: Math.min(...covered.map((w) => w.confidence)),
+            });
+        }
+    }
+    return matches;
+}

package/lib/src/services/omni-vision/screenScale.js

@@ -0,0 +1,68 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SAME_SCALE = void 0;
+exports.driverUsesPoints = driverUsesPoints;
+exports.screenScaleOf = screenScaleOf;
+exports.toDriverRect = toDriverRect;
+const sharp_1 = __importDefault(require("sharp"));
+exports.SAME_SCALE = { x: 1, y: 1 };
+/**
+ * Whether the driver's coordinates are points rather than the screenshot's
+ * pixels. A driver Xenon can't place (device control's screenshot-only
+ * stand-in, which never taps) keeps pixels.
+ */
+function driverUsesPoints(driver) {
+    var _a, _b, _c, _d;
+    const platform = String((_d = (_b = (_a = driver === null || driver === void 0 ? void 0 : driver.caps) === null || _a === void 0 ? void 0 : _a.platformName) !== null && _b !== void 0 ? _b : (_c = driver === null || driver === void 0 ? void 0 : driver.opts) === null || _c === void 0 ? void 0 : _c.platformName) !== null && _d !== void 0 ? _d : '');
+    return ['ios', 'tvos'].includes(platform.toLowerCase());
+}
+/**
+ * The scale between `screenshotBase64` and the driver's coordinates:
+ * 1 unless the driver uses points, then the screenshot's size over the
+ * window's. Throws when that can't be worked out on a driver that uses
+ * points, since a tap in pixels there lands in the wrong place.
+ */
+function screenScaleOf(driver, screenshotBase64) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!driverUsesPoints(driver))
+            return exports.SAME_SCALE;
+        const image = yield (0, sharp_1.default)(Buffer.from(screenshotBase64, 'base64'))
+            .metadata()
+            .catch((err) => {
+            throw new Error(`The screenshot's size could not be read: ${err.message}`);
+        });
+        const window = typeof driver.getWindowRect === 'function'
+            ? yield driver.getWindowRect()
+            : yield driver.getWindowSize();
+        const x = Number(image.width) / Number(window === null || window === void 0 ? void 0 : window.width);
+        const y = Number(image.height) / Number(window === null || window === void 0 ? void 0 : window.height);
+        if (!(Number.isFinite(x) && x > 0 && Number.isFinite(y) && y > 0)) {
+            throw new Error(`The screen size could not be worked out (screenshot ${image.width}x${image.height}, ` +
+                `window ${window === null || window === void 0 ? void 0 : window.width}x${window === null || window === void 0 ? void 0 : window.height}).`);
+        }
+        return { x, y };
+    });
+}
+/** A rect found in the screenshot, in the driver's coordinates. */
+function toDriverRect(rect, scale) {
+    if (scale.x === 1 && scale.y === 1)
+        return rect;
+    return {
+        x: rect.x / scale.x,
+        y: rect.y / scale.y,
+        width: rect.width / scale.x,
+        height: rect.height / scale.y,
+    };
+}