@wdio/ocr-service 1.0.0

package/dist/utils/imageProcessing.js

@@ -0,0 +1,73 @@
+import logger from '@wdio/logger';
+import Jimp from 'jimp';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const log = logger('@wdio/ocr-service:imageProcessing');
+export async function processImage({ contrast, elementRectangles, isAndroid, isIOS, ocrImagesPath, screenshot }) {
+    const image = await Jimp.read(Buffer.from(screenshot, 'base64'));
+    image.greyscale();
+    image.contrast(contrast);
+    const timestamp = new Date().getTime();
+    const platform = isAndroid ? 'android' : isIOS ? 'ios' : 'desktop';
+    let fileName = `${platform}-${timestamp}.png`;
+    let filePath = join(ocrImagesPath, fileName);
+    const screenshotSize = { width: image.bitmap.width, height: image.bitmap.height };
+    if (elementRectangles) {
+        // Make sure it doesn't crop outside of the screenshot
+        const cropX = Math.max(0, elementRectangles.x);
+        const cropY = Math.max(0, elementRectangles.y);
+        const cropWidth = Math.min(elementRectangles.width, screenshotSize.width - cropX);
+        const cropHeight = Math.min(elementRectangles.height, screenshotSize.height - cropY);
+        image.crop(cropX, cropY, cropWidth, cropHeight);
+        fileName = `${platform}-${timestamp}-cropped.png`;
+        filePath = join(ocrImagesPath, fileName);
+    }
+    await image.writeAsync(filePath);
+    return { filePath };
+}
+export async function drawTarget({ filePath, targetX, targetY }) {
+    try {
+        const targetImagePath = join(__dirname, '..', '..', 'assets', 'target.png');
+        const [sourceImage, targetImage] = await Promise.all([
+            Jimp.read(filePath),
+            Jimp.read(targetImagePath)
+        ]);
+        const xPosition = targetX - (targetImage.bitmap.width / 2);
+        const yPosition = targetY - (targetImage.bitmap.height / 2);
+        sourceImage.composite(targetImage, xPosition, yPosition);
+        await sourceImage.writeAsync(filePath);
+    }
+    catch (error) {
+        log.error('Failed to draw target on image:', error);
+    }
+}
+export async function drawHighlightedWords({ filePath, highlights }) {
+    try {
+        const image = await Jimp.read(filePath);
+        const highlightColor = { r: 57, g: 170, b: 86, a: 0.5 };
+        highlights.forEach(({ left, right, top, bottom }) => {
+            const width = right - left;
+            const height = bottom - top;
+            // Apply the semi-transparent highlight
+            for (let y = top; y < top + height; y++) {
+                for (let x = left; x < left + width; x++) {
+                    // Get the current pixel color
+                    const currentColor = image.getPixelColor(x, y);
+                    const rgba = Jimp.intToRGBA(currentColor);
+                    // Calculate new color values using simple alpha blending
+                    const newR = (highlightColor.r * highlightColor.a) + (rgba.r * (1 - highlightColor.a));
+                    const newG = (highlightColor.g * highlightColor.a) + (rgba.g * (1 - highlightColor.a));
+                    const newB = (highlightColor.b * highlightColor.a) + (rgba.b * (1 - highlightColor.a));
+                    const newA = rgba.a; // Use original alpha to maintain image integrity
+                    // Set the new pixel color
+                    image.setPixelColor(Jimp.rgbaToInt(newR, newG, newB, newA), x, y);
+                }
+            }
+        });
+        await image.writeAsync(filePath);
+    }
+    catch (error) {
+        log.error('Failed to highlight words on image:', error);
+    }
+}

package/dist/utils/index.d.ts

@@ -0,0 +1,15 @@
+/// <reference types="./standalone.js" />
+import type { ClickPoint, DetermineClickPointOptions, Rectangles, RectReturn, ScreenshotSize } from '../types.js';
+export declare function getScreenshotSize(screenshot: string): ScreenshotSize;
+export declare function getDprPositions(values: Rectangles, dpr: number): Rectangles;
+/**
+ * Determine the click point
+ */
+export declare function determineClickPoint(options: DetermineClickPointOptions): ClickPoint;
+/**
+ * Adjust the Element Bbox
+ */
+export declare function adjustElementBbox(bbox: Rectangles, elementRect: RectReturn): Rectangles;
+export declare function createOcrDir(imagesPath: string): string;
+export declare function isRectanglesObject(obj: WebdriverIO.Element | ChainablePromiseElement | RectReturn): boolean;
+//# sourceMappingURL=index.d.ts.map

package/dist/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,0BAA0B,EAAE,UAAU,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,cAAc,CAKpE;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,CAO3E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,0BAA0B,GAAG,UAAU,CAMnF;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,GAAG,UAAU,CAOvF;AAED,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAKvD;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,GAAG,uBAAuB,GAAG,UAAU,GAAG,OAAO,CAM3G"}

package/dist/utils/index.js

@@ -0,0 +1,46 @@
+import { join } from 'node:path';
+import { mkdirSync } from 'node:fs';
+export function getScreenshotSize(screenshot) {
+    return {
+        height: Math.round(Buffer.from(screenshot, 'base64').readUInt32BE(20)),
+        width: Math.round(Buffer.from(screenshot, 'base64').readUInt32BE(16)),
+    };
+}
+export function getDprPositions(values, dpr) {
+    Object.keys({ ...values }).map((value) => {
+        // @ts-ignore
+        values[value] /= dpr;
+    });
+    return values;
+}
+/**
+ * Determine the click point
+ */
+export function determineClickPoint(options) {
+    const { rectangles: { left, right, top, bottom } } = options;
+    const x = Math.round(left + (right - left) / 2);
+    const y = Math.round(top + (bottom - top) / 2);
+    return { x, y };
+}
+/**
+ * Adjust the Element Bbox
+ */
+export function adjustElementBbox(bbox, elementRect) {
+    return {
+        left: Math.round(bbox.left + elementRect.x),
+        top: Math.round(bbox.top + elementRect.y),
+        right: Math.round(bbox.right + elementRect.x),
+        bottom: Math.round(bbox.bottom + elementRect.y),
+    };
+}
+export function createOcrDir(imagesPath) {
+    const ocrDir = join(imagesPath, 'ocr');
+    mkdirSync(ocrDir, { recursive: true });
+    return ocrDir;
+}
+export function isRectanglesObject(obj) {
+    const properties = ['x', 'y', 'width', 'height'];
+    return obj !== null &&
+        typeof obj === 'object' &&
+        properties.every(prop => typeof obj[prop] === 'number' && !isNaN(obj[prop]) && obj[prop] >= 0);
+}

package/dist/utils/sendKeys.d.ts

@@ -0,0 +1,2 @@
+export default function sendKeys(value: string, submitValue: boolean): Promise<void>;
+//# sourceMappingURL=sendKeys.d.ts.map

package/dist/utils/sendKeys.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sendKeys.d.ts","sourceRoot":"","sources":["../../src/utils/sendKeys.ts"],"names":[],"mappings":"AAEA,wBAA8B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAqBzF"}

package/dist/utils/sendKeys.js

@@ -0,0 +1,21 @@
+import { Key } from 'webdriverio';
+export default async function sendKeys(value, submitValue) {
+    const actionChain = browser.action('key');
+    [...value].forEach((char) => {
+        actionChain.down(char);
+    });
+    [...value].forEach((char) => {
+        actionChain.up(char);
+    });
+    if (submitValue) {
+        /**
+         * XCTest API only allows to send keypresses (e.g. keydown+keyup).
+         */
+        if (!browser.isIOS) {
+            actionChain.pause(50);
+        }
+        actionChain.down(Key.Enter);
+        actionChain.up(Key.Enter);
+    }
+    await actionChain.perform();
+}

package/dist/utils/tesseract.d.ts

@@ -0,0 +1,13 @@
+import type { GetOcrData, ParseWordData, TessaractDataOptions } from '../types.js';
+export declare function isSystemTesseractAvailable(tesseractName?: string): boolean;
+export declare function parseWordDataFromText(attributes: string[]): ParseWordData;
+export declare function extractTextFromAlto(altoContent: string): string;
+/**
+ * Handle the OCR with Tesseract with pure JS
+ */
+export declare function getNodeOcrData(options: TessaractDataOptions): Promise<GetOcrData>;
+/**
+ * Handle the OCR with Tesseract with the system installed version
+ */
+export declare function getSystemOcrData(options: TessaractDataOptions): Promise<GetOcrData>;
+//# sourceMappingURL=tesseract.d.ts.map

package/dist/utils/tesseract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tesseract.d.ts","sourceRoot":"","sources":["../../src/utils/tesseract.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAQ,aAAa,EAAc,oBAAoB,EAAyD,MAAM,aAAa,CAAA;AAE3J,wBAAgB,0BAA0B,CAAC,aAAa,GAAE,MAAW,GAAG,OAAO,CAW9E;AAED,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,aAAa,CA4BzE;AAED,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAU/D;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,UAAU,CAAC,CA8FvF;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,UAAU,CAAC,CA8EzF"}

package/dist/utils/tesseract.js

@@ -0,0 +1,210 @@
+import { execSync } from 'node:child_process';
+import { createWorker, PSM } from 'tesseract.js';
+import { recognize } from 'node-tesseract-ocr';
+import { parseString } from 'xml2js';
+export function isSystemTesseractAvailable(tesseractName = '') {
+    const binary = tesseractName || 'tesseract';
+    const command = [binary, '--version'].join(' ');
+    try {
+        execSync(command);
+    }
+    catch (ign) {
+        return false;
+    }
+    return true;
+}
+export function parseWordDataFromText(attributes) {
+    let bbox = {
+        left: 0,
+        top: 0,
+        right: 0,
+        bottom: 0,
+    };
+    let wc = 0;
+    attributes.forEach((attribute) => {
+        if (attribute.includes('bbox')) {
+            const bboxValues = attribute.replace('bbox ', '').split(' ');
+            bbox = {
+                left: Number(bboxValues[0]),
+                top: Number(bboxValues[1]),
+                right: Number(bboxValues[2]),
+                bottom: Number(bboxValues[3]),
+            };
+        }
+        else if (attribute.includes('x_wconf')) {
+            const score = attribute.replace('x_wconf ', '');
+            wc = Number(score) / 100;
+        }
+    });
+    return {
+        ...{ bbox },
+        wc,
+    };
+}
+export function extractTextFromAlto(altoContent) {
+    const regex = /<String[^>]*CONTENT="([^"]+)"[^>]*>/g;
+    let match;
+    const textArray = [];
+    while ((match = regex.exec(altoContent)) !== null) {
+        textArray.push(match[1]);
+    }
+    return textArray.join(' ');
+}
+/**
+ * Handle the OCR with Tesseract with pure JS
+ */
+export async function getNodeOcrData(options) {
+    let worker;
+    try {
+        const { filePath, language } = options;
+        const jsonSingleWords = [];
+        const jsonWordStrings = [];
+        let composedBlocks = [];
+        worker = await createWorker(language);
+        await worker.setParameters({
+            tessedit_pageseg_mode: PSM.AUTO,
+            tessjs_create_tsv: '0',
+            tessjs_create_box: '0',
+            tessjs_create_unlv: '0',
+            tessjs_create_osd: '0',
+        });
+        const { data: { text, hocr } } = await worker.recognize(filePath);
+        const formatedText = text.replace(/[\r\n]{2,}/g, ' ').replace(/\s{2,}/g, ' ').trim();
+        await worker.terminate();
+        if (!hocr) {
+            throw Error(`No hocr data was found for the OCR, please verify image at: ${filePath}.`);
+        }
+        parseString(hocr, (error, data) => {
+            if (error) {
+                throw Error(`An error happened when parsing the getNodeOcrData, see: ${error}`);
+            }
+            composedBlocks = data.div.div;
+        });
+        if (!composedBlocks || composedBlocks.length === 0) {
+            throw Error('No text was found for the OCR, please verify the stored image.');
+        }
+        composedBlocks.forEach((block) => {
+            if (!block.p || !Array.isArray(block.p)) {
+                return;
+            }
+            block.p.forEach((paragraph) => {
+                if (!paragraph.span || !Array.isArray(paragraph.span)) {
+                    return;
+                }
+                paragraph.span.forEach((line) => {
+                    if (!line.span || !Array.isArray(line.span)) {
+                        return;
+                    }
+                    const lineData = {
+                        text: '',
+                        bbox: { bottom: 0, left: 0, right: 0, top: 0 },
+                    };
+                    line.span.forEach(word => {
+                        if (!word._ || !word.$) {
+                            return;
+                        }
+                        const text = word._;
+                        const title = word.$.title;
+                        const attributeValue = `; ${title}`.split('; ');
+                        const { bbox, wc } = parseWordDataFromText(attributeValue);
+                        jsonSingleWords.push({
+                            text,
+                            bbox,
+                            wc,
+                        });
+                        lineData.text = `${lineData.text} ${text}`.trim();
+                        lineData.bbox = bbox;
+                    });
+                    if (lineData.text !== '') {
+                        jsonWordStrings.push(lineData);
+                    }
+                });
+            });
+        });
+        return {
+            lines: jsonWordStrings,
+            words: jsonSingleWords,
+            text: formatedText,
+        };
+    }
+    catch (error) {
+        if (worker) {
+            await worker.terminate();
+        }
+        throw Error(`An error happened when parsing the getNodeOcrData, see: ${error}`);
+    }
+}
+/**
+ * Handle the OCR with Tesseract with the system installed version
+ */
+export async function getSystemOcrData(options) {
+    try {
+        const { filePath, language } = options;
+        const jsonSingleWords = [];
+        const jsonWordStrings = [];
+        let composedBlocks = [];
+        const result = await recognize(filePath, {
+            lang: language,
+            oem: 1,
+            // https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc
+            psm: 3,
+            presets: ['alto'],
+        });
+        parseString(result, (error, data) => {
+            if (error) {
+                throw Error(`An error happened when parsing the getSystemOcrData, see: ${error}`);
+            }
+            composedBlocks = data.alto.Layout[0].Page[0].PrintSpace[0].ComposedBlock;
+        });
+        const text = extractTextFromAlto(result);
+        if (!composedBlocks || composedBlocks.length === 0) {
+            throw Error('No text was found for the OCR, please verify the stored image.');
+        }
+        composedBlocks.forEach(composedBlock => {
+            composedBlock.TextBlock?.forEach(textBlock => {
+                textBlock.TextLine?.forEach(textLine => {
+                    let lineText = '';
+                    let lineBbox = null;
+                    if (textLine.$) {
+                        lineBbox = {
+                            left: Number(textLine.$.HPOS),
+                            top: Number(textLine.$.VPOS),
+                            right: Number(textLine.$.HPOS) + Number(textLine.$.WIDTH),
+                            bottom: Number(textLine.$.VPOS) + Number(textLine.$.HEIGHT),
+                        };
+                    }
+                    textLine.String?.forEach(string => {
+                        const { CONTENT, HPOS, VPOS, WIDTH, HEIGHT, WC } = string.$;
+                        if (!CONTENT || !HPOS || !VPOS || !WIDTH || !HEIGHT || !WC) {
+                            return;
+                        }
+                        const word = {
+                            text: CONTENT,
+                            bbox: {
+                                left: Number(HPOS),
+                                top: Number(VPOS),
+                                right: Number(HPOS) + Number(WIDTH),
+                                bottom: Number(VPOS) + Number(HEIGHT),
+                            },
+                            wc: Number(WC),
+                        };
+                        jsonSingleWords.push(word);
+                        lineText += `${CONTENT} `;
+                    });
+                    lineText = lineText.trim();
+                    if (lineText !== '' && lineBbox) {
+                        jsonWordStrings.push({ text: lineText, bbox: lineBbox });
+                    }
+                });
+            });
+        });
+        return {
+            lines: jsonWordStrings,
+            words: jsonSingleWords,
+            text: text,
+        };
+    }
+    catch (error) {
+        throw Error(`An error happened when parsing the getSystemOcrData, see: ${error}`);
+    }
+}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@wdio/ocr-service",
+  "author": "Wim Selles - wswebcreation",
+  "description": "A WebdriverIO service that is using Tesseract OCR for Desktop/Mobile Web and Mobile Native App tests.",
+  "version": "1.0.0",
+  "license": "MIT",
+  "homepage": "https://webdriver.io/docs/visual-testing",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/webdriverio/visual-testing.git"
+  },
+  "bin": {
+    "ocr-service": "./dist/cli.js"
+  },
+  "keywords": [
+    "webdriverio",
+    "tesseract",
+    "ocr",
+    "image"
+  ],
+  "exports": "./dist/index.js",
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "dependencies": {
+    "@wdio/globals": "^8.36.1",
+    "@wdio/logger": "^8.28.0",
+    "@wdio/types": "^8.36.1",
+    "fuse.js": "^7.0.0",
+    "inquirer": "^9.2.22",
+    "jimp": "^0.22.12",
+    "node-tesseract-ocr": "^2.2.1",
+    "tesseract.js": "^5.1.0",
+    "xml2js": "^0.6.2"
+  },
+  "scripts": {
+    "build": "run-s clean build:*",
+    "build:tsc": "tsc --project ./tsconfig.json",
+    "clean": "rimraf coverage build .tmp",
+    "watch": "pnpm run build:tsc -w"
+  }
+}