@aspiresys/visor 1.2.10 → 1.3.0

package/dist/index.d.ts

@@ -1,8 +1,8 @@
 import { Key, Button, Point } from "@nut-tree-fork/nut-js";
-import { Region } from "./types";
-export declare class Visor {
-    mouse: import("@nut-tree-fork/nut-js").MouseClass;
-    keyboard: import("@nut-tree-fork/nut-js").KeyboardClass;
+import { Offset, Region } from "./types";
+declare class Visor {
+    private mouse;
+    private keyboard;
     Key: typeof Key;
     Button: typeof Button;
     private configWarningShown;
@@ -32,10 +32,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    moveToRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    moveToRegion(region: Region, offset?: Offset): Promise<void>;
     /**
    * Performs a left mouse click at the
    * center of a specific screen region.
@@ -61,10 +58,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    clickRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    clickRegion(region: Region, offset?: Offset): Promise<void>;
     /**
    * Performs a double left mouse click
    * at the center of a specific
@@ -91,10 +85,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    doubleClickRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    doubleClickRegion(region: Region, offset?: Offset): Promise<void>;
     /**
    * Performs a right mouse click at the
    * center of a specific screen region.
@@ -120,10 +111,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    rightClickRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    rightClickRegion(region: Region, offset?: Offset): Promise<void>;
     /**
      * Finds an image on screen using OpenCV
      * and performs a left mouse click
@@ -154,10 +142,7 @@ export declare class Visor {
      * @example
      * await visor.click("./images/save.png", 0.9);
      */
-    click(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    click(image?: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Performs a right mouse click.
      *
@@ -182,10 +167,7 @@ export declare class Visor {
      * @example
      * await visor.rightClick();
      */
-    rightClick(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    rightClick(image?: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Performs a double left mouse click.
      *
@@ -211,10 +193,7 @@ export declare class Visor {
      * @example
      * await visor.doubleClick();
      */
-    doubleClick(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    doubleClick(image?: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Finds the best matching image on screen
      * using OpenCV template matching.
@@ -455,12 +434,7 @@ export declare class Visor {
      * const region =
      *   await visor.findText("Submit");
      */
-    findText(text: string, index?: number, region?: {
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-    }): Promise<Region>;
+    findText(text: string, index?: number, region?: Region): Promise<Region>;
     /**
      * Checks whether specific text exists
      * on screen using OCR.
@@ -474,12 +448,7 @@ export declare class Visor {
      * const exists =
      *   await visor.existsText("Teams");
      */
-    existsText(text: string, region?: {
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-    }): Promise<boolean>;
+    existsText(text: string, region?: Region): Promise<boolean>;
     /**
      * Performs OCR on a specific
      * screen region.
@@ -497,12 +466,7 @@ export declare class Visor {
      *     height: 300
      *   });
      */
-    readRegion(region: {
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-    }): Promise<any>;
+    readRegion(region: Region): Promise<any>;
     /**
      * Performs OCR on the entire screen
      * using Tesseract OCR.
@@ -622,10 +586,7 @@ export declare class Visor {
      * @example
      * await visor.hover("menu.png");
      */
-    hover(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    hover(image: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Scrolls the mouse wheel downward.
      *
@@ -719,11 +680,12 @@ export declare class Visor {
      *
      * @param x X coordinate.
      * @param y Y coordinate.
+     * @param inspectorCoords Boolean to represent if the coords are from visor-inspector
      *
      * @example
      * await visor.moveMouse(500, 300);
      */
-    moveMouse(x: number, y: number): Promise<void>;
+    moveMouse(x: number, y: number, inspectorCoords?: boolean): Promise<void>;
     /**
      * Returns the current mouse cursor
      * position.
@@ -1094,3 +1056,5 @@ export declare class Visor {
     input(message: string, title?: string): Promise<string>;
 }
 export declare const visor: Visor;
+export { Region } from "./types";
+export type { Offset } from "./types";

package/dist/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.visor = exports.Visor = void 0;
+exports.Region = exports.visor = void 0;
 const screen_1 = require("./screen");
 const matcher_1 = require("./matcher");
 const mouse_1 = require("./mouse");
@@ -73,10 +73,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async moveToRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async moveToRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
     }
     /**
@@ -104,10 +101,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async clickRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async clickRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
         await this.mouse.click(this.Button.LEFT);
     }
@@ -137,10 +131,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async doubleClickRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async doubleClickRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
         await this.mouse.doubleClick(this.Button.LEFT);
     }
@@ -169,10 +160,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async rightClickRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async rightClickRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
         await this.mouse.click(this.Button.RIGHT);
     }
@@ -206,10 +194,7 @@ class Visor {
      * @example
      * await visor.click("./images/save.png", 0.9);
      */
-    async click(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async click(image, confidence = 0.8, offset) {
         this.checkConfig();
         if (!image) {
             await this.mouse.click(this.Button.LEFT);
@@ -247,10 +232,7 @@ class Visor {
      * @example
      * await visor.rightClick();
      */
-    async rightClick(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async rightClick(image, confidence = 0.8, offset) {
         this.checkConfig();
         if (image) {
             const region = await this.find(image, confidence);
@@ -287,10 +269,7 @@ class Visor {
      * @example
      * await visor.doubleClick();
      */
-    async doubleClick(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async doubleClick(image, confidence = 0.8, offset) {
         this.checkConfig();
         if (image) {
             const region = await this.find(image, confidence);
@@ -336,8 +315,12 @@ class Visor {
         const start = Date.now();
         const screen = await (0, screen_1.captureScreen)();
         const template = await (0, matcher_1.loadTemplate)((0, path_1.resolveImagePath)(image));
+        const scaleFactor = config_1.visorConfig.scaleFactor;
         try {
             const result = (0, matcher_1.matchTemplate)(screen, template, confidence);
+            if (result) {
+                (0, logger_1.log)(`[FIND] Actual coords: (${Math.floor(result.x * scaleFactor)}, ${Math.floor(result.y * scaleFactor)})`);
+            }
             (0, logger_1.log)(`[FIND] Total: ${Date.now() - start} ms`);
             return result;
         }
@@ -811,10 +794,7 @@ CONF:${m.confidence.toFixed(3)}
      * @example
      * await visor.hover("menu.png");
      */
-    async hover(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async hover(image, confidence = 0.8, offset) {
         this.checkConfig();
         const region = await this.find(image, confidence);
         if (!region) {
@@ -929,13 +909,19 @@ CONF:${m.confidence.toFixed(3)}
      *
      * @param x X coordinate.
      * @param y Y coordinate.
+     * @param inspectorCoords Boolean to represent if the coords are from visor-inspector
      *
      * @example
      * await visor.moveMouse(500, 300);
      */
-    async moveMouse(x, y) {
+    async moveMouse(x, y, inspectorCoords = false) {
         this.checkConfig();
-        await this.mouse.move((0, nut_js_1.straightTo)(new nut_js_1.Point(x, y)));
+        if (inspectorCoords) {
+            await this.mouse.move((0, nut_js_1.straightTo)(new nut_js_1.Point(x, y)));
+            return;
+        }
+        const scaleFactor = config_1.visorConfig.scaleFactor;
+        await this.mouse.move((0, nut_js_1.straightTo)(new nut_js_1.Point(Math.floor(x / scaleFactor), Math.floor(y / scaleFactor))));
     }
     /**
      * Returns the current mouse cursor
@@ -1402,5 +1388,6 @@ CONF:${m.confidence.toFixed(3)}
         return await (0, dialog_1.prompt)(message, title);
     }
 }
-exports.Visor = Visor;
 exports.visor = new Visor();
+var types_1 = require("./types");
+Object.defineProperty(exports, "Region", { enumerable: true, get: function () { return types_1.Region; } });

package/dist/matcher.d.ts

@@ -3,3 +3,6 @@ export declare function loadTemplate(path: string): Promise<any>;
 export declare function loadScreen(path: string): Promise<any>;
 export declare function matchTemplate(screen: any, template: any, confidence?: number): Region | null;
 export declare function findAllMatches(screen: any, template: any, confidence?: number): Region[];
+export declare function findAllInspectorMatches(screen: any, template: any, confidence?: number): Region[];
+export declare function matchInspectorTemplate(screen: any, template: any, confidence?: number, multiScale?: boolean): Region | null;
+export declare function cropMat(screen: any, region: Region): any;

package/dist/matcher.js

@@ -7,10 +7,14 @@ exports.loadTemplate = loadTemplate;
 exports.loadScreen = loadScreen;
 exports.matchTemplate = matchTemplate;
 exports.findAllMatches = findAllMatches;
+exports.findAllInspectorMatches = findAllInspectorMatches;
+exports.matchInspectorTemplate = matchInspectorTemplate;
+exports.cropMat = cropMat;
 const fs_1 = __importDefault(require("fs"));
 const pngjs_1 = require("pngjs");
 const cv = require("@techstark/opencv-js");
 const templateCache = new Map();
+const types_1 = require("./types");
 const logger_1 = require("./logger");
 async function loadTemplate(path) {
     if (templateCache.has(path)) {
@@ -62,13 +66,7 @@ function matchTemplate(screen, template, confidence = 0.8) {
         (0, logger_1.log)(`[MATCH] Scale ${scale} -> ${maxVal}`);
         result.delete();
         if (maxVal >= confidence && (!bestMatch || maxVal > bestMatch.confidence)) {
-            bestMatch = {
-                x: maxLoc.x,
-                y: maxLoc.y,
-                width: resizedTemplate.cols,
-                height: resizedTemplate.rows,
-                confidence: maxVal,
-            };
+            bestMatch = new types_1.Region(maxLoc.x, maxLoc.y, resizedTemplate.cols, resizedTemplate.rows, maxVal);
             if (maxVal >= 0.98) {
                 resizedTemplate.delete();
                 (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
@@ -88,13 +86,7 @@ function findAllMatches(screen, template, confidence = 0.8) {
         for (let x = 0; x < result.cols; x++) {
             const score = result.floatAt(y, x);
             if (score >= confidence) {
-                matches.push({
-                    x,
-                    y,
-                    width: template.cols,
-                    height: template.rows,
-                    confidence: score,
-                });
+                matches.push(new types_1.Region(x, y, template.cols, template.rows, score));
             }
         }
     }
@@ -113,3 +105,83 @@ function findAllMatches(screen, template, confidence = 0.8) {
     result.delete();
     return filtered;
 }
+function findAllInspectorMatches(screen, template, confidence = 0.8) {
+    const result = new cv.Mat();
+    cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
+    const matches = [];
+    for (let y = 0; y < result.rows; y++) {
+        for (let x = 0; x < result.cols; x++) {
+            const score = result.floatAt(y, x);
+            if (score >= confidence) {
+                matches.push(new types_1.Region(x, y, template.cols, template.rows, score));
+            }
+        }
+    }
+    const filtered = [];
+    for (const match of matches) {
+        const tooClose = filtered.some((existing) => {
+            const dx = Math.abs(existing.x - match.x);
+            const dy = Math.abs(existing.y - match.y);
+            return dx < 10 && dy < 10;
+        });
+        if (!tooClose) {
+            filtered.push(match);
+        }
+    }
+    result.delete();
+    return filtered;
+}
+function matchInspectorTemplate(screen, template, confidence = 0.8, multiScale = false) {
+    if (multiScale) {
+        const start = Date.now();
+        const scales = [1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3, 0.6, 1.4, 0.5, 1.5];
+        let bestMatch = null;
+        for (const scale of scales) {
+            const resizedTemplate = resizeTemplate(template, scale);
+            if (resizedTemplate.cols > screen.cols ||
+                resizedTemplate.rows > screen.rows) {
+                resizedTemplate.delete();
+                continue;
+            }
+            const result = new cv.Mat();
+            cv.matchTemplate(screen, resizedTemplate, result, cv.TM_CCOEFF_NORMED);
+            const { maxLoc, maxVal } = cv.minMaxLoc(result);
+            (0, logger_1.log)(`[MATCH] Scale ${scale} -> ${maxVal}`);
+            result.delete();
+            if (maxVal >= confidence && (!bestMatch || maxVal > bestMatch.confidence)) {
+                bestMatch = new types_1.Region(maxLoc.x, maxLoc.y, resizedTemplate.cols, resizedTemplate.rows, maxVal);
+                if (maxVal >= 0.98) {
+                    resizedTemplate.delete();
+                    (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
+                    return bestMatch;
+                }
+            }
+            resizedTemplate.delete();
+        }
+        (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
+        return bestMatch;
+    }
+    else {
+        const start = Date.now();
+        const result = new cv.Mat();
+        cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
+        const { maxLoc, maxVal } = cv.minMaxLoc(result);
+        result.delete();
+        if (maxVal >= confidence) {
+            return new types_1.Region(maxLoc.x, maxLoc.y, template.cols, template.rows, maxVal);
+        }
+        (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
+        return null;
+    }
+}
+function cropMat(screen, region) {
+    const x = Math.max(0, Math.floor(region.x));
+    const y = Math.max(0, Math.floor(region.y));
+    const width = Math.min(Math.floor(region.width), screen.cols - x);
+    const height = Math.min(Math.floor(region.height), screen.rows - y);
+    const rect = new cv.Rect(x, y, width, height);
+    const croppedMat = screen.roi(rect);
+    const isolatedMat = croppedMat.clone();
+    croppedMat.delete();
+    return isolatedMat;
+}

package/dist/mouse.d.ts

@@ -1,9 +1,3 @@
-import { Region } from "./types";
-export declare function moveToRegion(region: Region, offset?: {
-    x: number;
-    y: number;
-}): Promise<void>;
-export declare function clickRegion(region: Region, offset?: {
-    x: number;
-    y: number;
-}): Promise<void>;
+import { Offset, Region } from "./types";
+export declare function moveToRegion(region: Region, offset?: Offset): Promise<void>;
+export declare function clickRegion(region: Region, offset?: Offset): Promise<void>;

package/dist/mouse.js

@@ -10,8 +10,9 @@ async function moveToRegion(region, offset = {
     y: 0
 }) {
     const scaleFactor = config_1.visorConfig.scaleFactor;
-    const centerX = Math.floor((region.x + region.width / 2 + offset.x) / scaleFactor);
-    const centerY = Math.floor((region.y + region.height / 2 + offset.y) / scaleFactor);
+    const center = region.center();
+    const centerX = Math.floor((center.x + offset.x) / scaleFactor);
+    const centerY = Math.floor((center.y + offset.y) / scaleFactor);
     (0, logger_1.log)("[MOUSE] Moving mouse to:", centerX, centerY);
     await nut_js_1.mouse.move([new nut_js_1.Point(centerX, centerY)]);
 }

package/dist/ocr.d.ts

@@ -1,3 +1,4 @@
+import { Region } from "./types";
 /**
  * Terminates OCR worker manually.
  *
@@ -15,9 +16,4 @@ export declare function terminateOCR(): Promise<void>;
  * - sharp preprocessing
  * - Tesseract OCR
  */
-export declare function extractTextFromRegion(region?: {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}): Promise<any>;
+export declare function extractTextFromRegion(region?: Region): Promise<any>;

package/dist/region.d.ts

@@ -0,0 +1,9 @@
+export declare class Region {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    confidence: number;
+    constructor(x: number, y: number, width: number, height: number, confidence?: number);
+    center(): void;
+}

package/dist/region.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Region = void 0;
+class Region {
+    constructor(x, y, width, height, confidence = 1) {
+        this.x = x;
+        this.y = y;
+        this.width = width;
+        this.height = height;
+        this.confidence = confidence;
+    }
+    center() {
+    }
+}
+exports.Region = Region;

package/dist/text.d.ts

@@ -1,13 +1,3 @@
 import { Region } from "./types";
-export declare function findText(text: string, index?: number, region?: {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}): Promise<Region | null>;
-export declare function existsText(text: string, region?: {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}): Promise<boolean>;
+export declare function findText(text: string, index?: number, region?: Region): Promise<Region | null>;
+export declare function existsText(text: string, region?: Region): Promise<boolean>;

package/dist/text.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.findText = findText;
 exports.existsText = existsText;
 const ocr_1 = require("./ocr");
+const types_1 = require("./types");
 async function findText(text, index = 0, region) {
     const result = await (0, ocr_1.extractTextFromRegion)(region);
     const tsv = result.tsv;
@@ -62,13 +63,7 @@ async function findText(text, index = 0, region) {
             const maxX = Math.max(...regionWords.map((w) => w.x + w.width));
             const maxY = Math.max(...regionWords.map((w) => w.y + w.height));
             const avgConfidence = words.reduce((sum, w) => sum + w.confidence, 0) / words.length;
-            matches.push({
-                x: minX,
-                y: minY,
-                width: maxX - minX,
-                height: maxY - minY,
-                confidence: avgConfidence,
-            });
+            matches.push(new types_1.Region(minX, minY, maxX - minX, maxY - minY, avgConfidence));
         }
     }
     matches.sort((a, b) => a.y - b.y || a.x - b.x);