@aspiresys/visor 1.2.11 → 1.3.0

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { Key, Button, Point } from "@nut-tree-fork/nut-js";
-import { Region } from "./types";
-export declare class Visor {
+import { Offset, Region } from "./types";
+declare class Visor {
     private mouse;
     private keyboard;
     Key: typeof Key;
@@ -32,10 +32,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    moveToRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    moveToRegion(region: Region, offset?: Offset): Promise<void>;
     /**
    * Performs a left mouse click at the
    * center of a specific screen region.
@@ -61,10 +58,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    clickRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    clickRegion(region: Region, offset?: Offset): Promise<void>;
     /**
    * Performs a double left mouse click
    * at the center of a specific
@@ -91,10 +85,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    doubleClickRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    doubleClickRegion(region: Region, offset?: Offset): Promise<void>;
     /**
    * Performs a right mouse click at the
    * center of a specific screen region.
@@ -120,10 +111,7 @@ export declare class Visor {
    *   height: 50
    * });
    */
-    rightClickRegion(region: Region, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    rightClickRegion(region: Region, offset?: Offset): Promise<void>;
     /**
      * Finds an image on screen using OpenCV
      * and performs a left mouse click
@@ -154,10 +142,7 @@ export declare class Visor {
      * @example
      * await visor.click("./images/save.png", 0.9);
      */
-    click(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    click(image?: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Performs a right mouse click.
      *
@@ -182,10 +167,7 @@ export declare class Visor {
      * @example
      * await visor.rightClick();
      */
-    rightClick(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    rightClick(image?: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Performs a double left mouse click.
      *
@@ -211,10 +193,7 @@ export declare class Visor {
      * @example
      * await visor.doubleClick();
      */
-    doubleClick(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    doubleClick(image?: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Finds the best matching image on screen
      * using OpenCV template matching.
@@ -455,12 +434,7 @@ export declare class Visor {
      * const region =
      *   await visor.findText("Submit");
      */
-    findText(text: string, index?: number, region?: {
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-    }): Promise<Region>;
+    findText(text: string, index?: number, region?: Region): Promise<Region>;
     /**
      * Checks whether specific text exists
      * on screen using OCR.
@@ -474,12 +448,7 @@ export declare class Visor {
      * const exists =
      *   await visor.existsText("Teams");
      */
-    existsText(text: string, region?: {
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-    }): Promise<boolean>;
+    existsText(text: string, region?: Region): Promise<boolean>;
     /**
      * Performs OCR on a specific
      * screen region.
@@ -497,12 +466,7 @@ export declare class Visor {
      *     height: 300
      *   });
      */
-    readRegion(region: {
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-    }): Promise<any>;
+    readRegion(region: Region): Promise<any>;
     /**
      * Performs OCR on the entire screen
      * using Tesseract OCR.
@@ -622,10 +586,7 @@ export declare class Visor {
      * @example
      * await visor.hover("menu.png");
      */
-    hover(image?: string, confidence?: number, offset?: {
-        x: number;
-        y: number;
-    }): Promise<void>;
+    hover(image: string, confidence?: number, offset?: Offset): Promise<void>;
     /**
      * Scrolls the mouse wheel downward.
      *
@@ -1095,3 +1056,5 @@ export declare class Visor {
     input(message: string, title?: string): Promise<string>;
 }
 export declare const visor: Visor;
+export { Region } from "./types";
+export type { Offset } from "./types";

package/dist/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.visor = exports.Visor = void 0;
+exports.Region = exports.visor = void 0;
 const screen_1 = require("./screen");
 const matcher_1 = require("./matcher");
 const mouse_1 = require("./mouse");
@@ -73,10 +73,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async moveToRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async moveToRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
     }
     /**
@@ -104,10 +101,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async clickRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async clickRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
         await this.mouse.click(this.Button.LEFT);
     }
@@ -137,10 +131,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async doubleClickRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async doubleClickRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
         await this.mouse.doubleClick(this.Button.LEFT);
     }
@@ -169,10 +160,7 @@ class Visor {
    *   height: 50
    * });
    */
-    async rightClickRegion(region, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async rightClickRegion(region, offset) {
         await (0, mouse_1.moveToRegion)(region, offset);
         await this.mouse.click(this.Button.RIGHT);
     }
@@ -206,10 +194,7 @@ class Visor {
      * @example
      * await visor.click("./images/save.png", 0.9);
      */
-    async click(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async click(image, confidence = 0.8, offset) {
         this.checkConfig();
         if (!image) {
             await this.mouse.click(this.Button.LEFT);
@@ -247,10 +232,7 @@ class Visor {
      * @example
      * await visor.rightClick();
      */
-    async rightClick(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async rightClick(image, confidence = 0.8, offset) {
         this.checkConfig();
         if (image) {
             const region = await this.find(image, confidence);
@@ -287,10 +269,7 @@ class Visor {
      * @example
      * await visor.doubleClick();
      */
-    async doubleClick(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async doubleClick(image, confidence = 0.8, offset) {
         this.checkConfig();
         if (image) {
             const region = await this.find(image, confidence);
@@ -815,10 +794,7 @@ CONF:${m.confidence.toFixed(3)}
      * @example
      * await visor.hover("menu.png");
      */
-    async hover(image, confidence = 0.8, offset = {
-        x: 0,
-        y: 0
-    }) {
+    async hover(image, confidence = 0.8, offset) {
         this.checkConfig();
         const region = await this.find(image, confidence);
         if (!region) {
@@ -1412,5 +1388,6 @@ CONF:${m.confidence.toFixed(3)}
         return await (0, dialog_1.prompt)(message, title);
     }
 }
-exports.Visor = Visor;
 exports.visor = new Visor();
+var types_1 = require("./types");
+Object.defineProperty(exports, "Region", { enumerable: true, get: function () { return types_1.Region; } });

package/dist/matcher.d.ts

@@ -3,3 +3,6 @@ export declare function loadTemplate(path: string): Promise<any>;
 export declare function loadScreen(path: string): Promise<any>;
 export declare function matchTemplate(screen: any, template: any, confidence?: number): Region | null;
 export declare function findAllMatches(screen: any, template: any, confidence?: number): Region[];
+export declare function findAllInspectorMatches(screen: any, template: any, confidence?: number): Region[];
+export declare function matchInspectorTemplate(screen: any, template: any, confidence?: number, multiScale?: boolean): Region | null;
+export declare function cropMat(screen: any, region: Region): any;

package/dist/matcher.js

@@ -7,10 +7,14 @@ exports.loadTemplate = loadTemplate;
 exports.loadScreen = loadScreen;
 exports.matchTemplate = matchTemplate;
 exports.findAllMatches = findAllMatches;
+exports.findAllInspectorMatches = findAllInspectorMatches;
+exports.matchInspectorTemplate = matchInspectorTemplate;
+exports.cropMat = cropMat;
 const fs_1 = __importDefault(require("fs"));
 const pngjs_1 = require("pngjs");
 const cv = require("@techstark/opencv-js");
 const templateCache = new Map();
+const types_1 = require("./types");
 const logger_1 = require("./logger");
 async function loadTemplate(path) {
     if (templateCache.has(path)) {
@@ -62,13 +66,7 @@ function matchTemplate(screen, template, confidence = 0.8) {
         (0, logger_1.log)(`[MATCH] Scale ${scale} -> ${maxVal}`);
         result.delete();
         if (maxVal >= confidence && (!bestMatch || maxVal > bestMatch.confidence)) {
-            bestMatch = {
-                x: maxLoc.x,
-                y: maxLoc.y,
-                width: resizedTemplate.cols,
-                height: resizedTemplate.rows,
-                confidence: maxVal,
-            };
+            bestMatch = new types_1.Region(maxLoc.x, maxLoc.y, resizedTemplate.cols, resizedTemplate.rows, maxVal);
             if (maxVal >= 0.98) {
                 resizedTemplate.delete();
                 (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
@@ -88,13 +86,7 @@ function findAllMatches(screen, template, confidence = 0.8) {
         for (let x = 0; x < result.cols; x++) {
             const score = result.floatAt(y, x);
             if (score >= confidence) {
-                matches.push({
-                    x,
-                    y,
-                    width: template.cols,
-                    height: template.rows,
-                    confidence: score,
-                });
+                matches.push(new types_1.Region(x, y, template.cols, template.rows, score));
             }
         }
     }
@@ -113,3 +105,83 @@ function findAllMatches(screen, template, confidence = 0.8) {
     result.delete();
     return filtered;
 }
+function findAllInspectorMatches(screen, template, confidence = 0.8) {
+    const result = new cv.Mat();
+    cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
+    const matches = [];
+    for (let y = 0; y < result.rows; y++) {
+        for (let x = 0; x < result.cols; x++) {
+            const score = result.floatAt(y, x);
+            if (score >= confidence) {
+                matches.push(new types_1.Region(x, y, template.cols, template.rows, score));
+            }
+        }
+    }
+    const filtered = [];
+    for (const match of matches) {
+        const tooClose = filtered.some((existing) => {
+            const dx = Math.abs(existing.x - match.x);
+            const dy = Math.abs(existing.y - match.y);
+            return dx < 10 && dy < 10;
+        });
+        if (!tooClose) {
+            filtered.push(match);
+        }
+    }
+    result.delete();
+    return filtered;
+}
+function matchInspectorTemplate(screen, template, confidence = 0.8, multiScale = false) {
+    if (multiScale) {
+        const start = Date.now();
+        const scales = [1.0, 0.9, 1.1, 0.8, 1.2, 0.7, 1.3, 0.6, 1.4, 0.5, 1.5];
+        let bestMatch = null;
+        for (const scale of scales) {
+            const resizedTemplate = resizeTemplate(template, scale);
+            if (resizedTemplate.cols > screen.cols ||
+                resizedTemplate.rows > screen.rows) {
+                resizedTemplate.delete();
+                continue;
+            }
+            const result = new cv.Mat();
+            cv.matchTemplate(screen, resizedTemplate, result, cv.TM_CCOEFF_NORMED);
+            const { maxLoc, maxVal } = cv.minMaxLoc(result);
+            (0, logger_1.log)(`[MATCH] Scale ${scale} -> ${maxVal}`);
+            result.delete();
+            if (maxVal >= confidence && (!bestMatch || maxVal > bestMatch.confidence)) {
+                bestMatch = new types_1.Region(maxLoc.x, maxLoc.y, resizedTemplate.cols, resizedTemplate.rows, maxVal);
+                if (maxVal >= 0.98) {
+                    resizedTemplate.delete();
+                    (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
+                    return bestMatch;
+                }
+            }
+            resizedTemplate.delete();
+        }
+        (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
+        return bestMatch;
+    }
+    else {
+        const start = Date.now();
+        const result = new cv.Mat();
+        cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
+        const { maxLoc, maxVal } = cv.minMaxLoc(result);
+        result.delete();
+        if (maxVal >= confidence) {
+            return new types_1.Region(maxLoc.x, maxLoc.y, template.cols, template.rows, maxVal);
+        }
+        (0, logger_1.log)(`[MATCH] Total: ${Date.now() - start} ms`);
+        return null;
+    }
+}
+function cropMat(screen, region) {
+    const x = Math.max(0, Math.floor(region.x));
+    const y = Math.max(0, Math.floor(region.y));
+    const width = Math.min(Math.floor(region.width), screen.cols - x);
+    const height = Math.min(Math.floor(region.height), screen.rows - y);
+    const rect = new cv.Rect(x, y, width, height);
+    const croppedMat = screen.roi(rect);
+    const isolatedMat = croppedMat.clone();
+    croppedMat.delete();
+    return isolatedMat;
+}

package/dist/mouse.d.ts

@@ -1,9 +1,3 @@
-import { Region } from "./types";
-export declare function moveToRegion(region: Region, offset?: {
-    x: number;
-    y: number;
-}): Promise<void>;
-export declare function clickRegion(region: Region, offset?: {
-    x: number;
-    y: number;
-}): Promise<void>;
+import { Offset, Region } from "./types";
+export declare function moveToRegion(region: Region, offset?: Offset): Promise<void>;
+export declare function clickRegion(region: Region, offset?: Offset): Promise<void>;

package/dist/mouse.js

@@ -10,8 +10,9 @@ async function moveToRegion(region, offset = {
     y: 0
 }) {
     const scaleFactor = config_1.visorConfig.scaleFactor;
-    const centerX = Math.floor((region.x + region.width / 2 + offset.x) / scaleFactor);
-    const centerY = Math.floor((region.y + region.height / 2 + offset.y) / scaleFactor);
+    const center = region.center();
+    const centerX = Math.floor((center.x + offset.x) / scaleFactor);
+    const centerY = Math.floor((center.y + offset.y) / scaleFactor);
     (0, logger_1.log)("[MOUSE] Moving mouse to:", centerX, centerY);
     await nut_js_1.mouse.move([new nut_js_1.Point(centerX, centerY)]);
 }

package/dist/ocr.d.ts

@@ -1,3 +1,4 @@
+import { Region } from "./types";
 /**
  * Terminates OCR worker manually.
  *
@@ -15,9 +16,4 @@ export declare function terminateOCR(): Promise<void>;
  * - sharp preprocessing
  * - Tesseract OCR
  */
-export declare function extractTextFromRegion(region?: {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}): Promise<any>;
+export declare function extractTextFromRegion(region?: Region): Promise<any>;

package/dist/region.d.ts

@@ -0,0 +1,9 @@
+export declare class Region {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    confidence: number;
+    constructor(x: number, y: number, width: number, height: number, confidence?: number);
+    center(): void;
+}

package/dist/region.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Region = void 0;
+class Region {
+    constructor(x, y, width, height, confidence = 1) {
+        this.x = x;
+        this.y = y;
+        this.width = width;
+        this.height = height;
+        this.confidence = confidence;
+    }
+    center() {
+    }
+}
+exports.Region = Region;

package/dist/text.d.ts

@@ -1,13 +1,3 @@
 import { Region } from "./types";
-export declare function findText(text: string, index?: number, region?: {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}): Promise<Region | null>;
-export declare function existsText(text: string, region?: {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}): Promise<boolean>;
+export declare function findText(text: string, index?: number, region?: Region): Promise<Region | null>;
+export declare function existsText(text: string, region?: Region): Promise<boolean>;

package/dist/text.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.findText = findText;
 exports.existsText = existsText;
 const ocr_1 = require("./ocr");
+const types_1 = require("./types");
 async function findText(text, index = 0, region) {
     const result = await (0, ocr_1.extractTextFromRegion)(region);
     const tsv = result.tsv;
@@ -62,13 +63,7 @@ async function findText(text, index = 0, region) {
             const maxX = Math.max(...regionWords.map((w) => w.x + w.width));
             const maxY = Math.max(...regionWords.map((w) => w.y + w.height));
             const avgConfidence = words.reduce((sum, w) => sum + w.confidence, 0) / words.length;
-            matches.push({
-                x: minX,
-                y: minY,
-                width: maxX - minX,
-                height: maxY - minY,
-                confidence: avgConfidence,
-            });
+            matches.push(new types_1.Region(minX, minY, maxX - minX, maxY - minY, avgConfidence));
         }
     }
     matches.sort((a, b) => a.y - b.y || a.x - b.x);

package/dist/types.d.ts

@@ -1,10 +1,174 @@
-export interface Region {
+/**
+ * Represents a rectangular screen region.
+ *
+ * A Region can be returned by:
+ * - image matching
+ * - OCR text search
+ * - region-based searches
+ *
+ * Provides helper methods for:
+ * - mouse interaction
+ * - OCR operations
+ * - image matching
+ * - geometry calculations
+ * Example:
+ *
+ * const dialog = await visor.find("dialog.png");
+ *
+ * const saveButton = await dialog.find("save.png");
+ *
+ * await saveButton.click();
+ *
+ * const text = await dialog.findText("Submit");
+ */
+export declare class Region {
     x: number;
     y: number;
     width: number;
     height: number;
     confidence: number;
+    private static configWarningShown;
+    constructor(x: number, y: number, width: number, height: number, confidence?: number);
+    private checkConfig;
+    /**
+   * Returns the center point of the region.
+   *
+   * Useful for:
+   * - mouse movement
+   * - coordinate calculations
+   * - custom interactions
+   */
+    center(): {
+        x: number;
+        y: number;
+    };
+    /**
+   * Moves the mouse to the center of the region.
+   *
+   * Mouse coordinates are automatically
+   * adjusted for display scaling.
+   */
+    move(): Promise<void>;
+    /**
+   * Performs a left click on the region.
+   *
+   * Clicks the center of the region by default.
+   *
+   * Optional offset can be supplied to click
+   * a specific position relative to the center.
+   */
+    click(offset?: Offset): Promise<void>;
+    /**
+   * Performs a double left click on the region.
+   *
+   * Optional offset can be supplied to click
+   * a specific position relative to the center.
+   */
+    doubleClick(offset?: Offset): Promise<void>;
+    /**
+   * Performs a right click on the region.
+   *
+   * Optional offset can be supplied to click
+   * a specific position relative to the center.
+   */
+    rightClick(offset?: Offset): Promise<void>;
+    /**
+   * Checks whether the supplied coordinates
+   * are located inside this region.
+   *
+   * Returns:
+   * - true if inside
+   * - false otherwise
+   */
+    contains(x: number, y: number): boolean;
+    /**
+   * Creates a new Region shifted by the
+   * supplied x and y offsets.
+   *
+   * Original region remains unchanged.
+   */
+    offset(x: number, y: number): Region;
+    /**
+   * Checks whether this region overlaps
+   * with another region.
+   *
+   * Returns:
+   * - true if regions intersect
+   * - false otherwise
+   */
+    intersects(region: Region): boolean;
+    /**
+   * Returns the total area of the region
+   * in pixels.
+   */
+    area(): number;
+    /**
+   * Extracts OCR text from this region.
+   *
+   * Returns full OCR output including:
+   * - text
+   * - TSV data
+   * - HOCR data
+   * - block information
+   */
+    readText(): Promise<any>;
+    /**
+   * Searches for text within this region.
+   *
+   * Returns the matching text region
+   * if found, otherwise null.
+   *
+   * Supports selecting a specific match
+   * using the index parameter.
+   */
+    findText(text: string, index?: number): Promise<Region>;
+    /**
+   * Checks whether the supplied text
+   * exists within this region.
+   *
+   * Returns:
+   * - true if found
+   * - false otherwise
+   */
+    existsText(text: string): Promise<boolean>;
+    /**
+   * Searches for an image within this region.
+   *
+   * Matching is performed only inside
+   * the current region, improving speed
+   * and reducing false positives.
+   *
+   * Returns:
+   * - matching Region if found
+   * - null otherwise
+   */
+    find(image: string, confidence?: number): Promise<Region>;
+    /**
+   * Finds all image matches within this region.
+   *
+   * Matching is performed only inside
+   * the current region.
+   *
+   * Returns an array of matching regions.
+   */
+    findAll(image: string, confidence?: number): Promise<Region[]>;
+    /**
+   * Checks whether an image exists
+   * within this region.
+   *
+   * Returns:
+   * - true if found
+   * - false otherwise
+   */
+    exists(image: string, confidence?: number): Promise<boolean>;
 }
+/**
+ * Represents an offset relative to
+ * a region's center point.
+ *
+ * Useful for clicking specific locations
+ * within a matched region.
+ */
 export interface Offset {
     x: number;
     y: number;