@aspiresys/visor 1.0.0

package/dist/logger.d.ts

@@ -0,0 +1 @@
+export declare function log(...args: any[]): void;

package/dist/logger.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.log = log;
+const config_1 = require("./config");
+function log(...args) {
+    if (config_1.visorConfig.debug) {
+        console.log(...args);
+    }
+}

package/dist/matcher.d.ts

@@ -0,0 +1,4 @@
+import { Region } from "./types";
+export declare function loadTemplate(path: string): any;
+export declare function matchTemplate(screen: any, template: any, confidence?: number): Region | null;
+export declare function findAllMatches(screen: any, template: any, confidence?: number): Region[];

package/dist/matcher.js

@@ -0,0 +1,69 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadTemplate = loadTemplate;
+exports.matchTemplate = matchTemplate;
+exports.findAllMatches = findAllMatches;
+const fs_1 = __importDefault(require("fs"));
+const pngjs_1 = require("pngjs");
+const cv = require("@techstark/opencv-js");
+function loadTemplate(path) {
+    const buffer = fs_1.default.readFileSync(path);
+    const png = pngjs_1.PNG.sync.read(buffer);
+    return cv.matFromImageData({
+        data: png.data,
+        width: png.width,
+        height: png.height
+    });
+}
+function matchTemplate(screen, template, confidence = 0.8) {
+    const result = new cv.Mat();
+    cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
+    const { maxLoc, maxVal } = cv.minMaxLoc(result);
+    result.delete();
+    if (maxVal < confidence) {
+        return null;
+    }
+    return {
+        x: maxLoc.x,
+        y: maxLoc.y,
+        width: template.cols,
+        height: template.rows,
+        confidence: maxVal
+    };
+}
+function findAllMatches(screen, template, confidence = 0.8) {
+    const result = new cv.Mat();
+    cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
+    const matches = [];
+    for (let y = 0; y < result.rows; y++) {
+        for (let x = 0; x < result.cols; x++) {
+            const score = result.floatAt(y, x);
+            if (score >= confidence) {
+                matches.push({
+                    x,
+                    y,
+                    width: template.cols,
+                    height: template.rows,
+                    confidence: score
+                });
+            }
+        }
+    }
+    matches.sort((a, b) => b.confidence - a.confidence);
+    const filtered = [];
+    for (const match of matches) {
+        const tooClose = filtered.some(existing => {
+            const dx = Math.abs(existing.x - match.x);
+            const dy = Math.abs(existing.y - match.y);
+            return dx < 10 && dy < 10;
+        });
+        if (!tooClose) {
+            filtered.push(match);
+        }
+    }
+    result.delete();
+    return filtered;
+}

package/dist/mouse.d.ts

@@ -0,0 +1,3 @@
+import { Region } from "./types";
+export declare function moveToRegion(region: Region): Promise<void>;
+export declare function clickRegion(region: Region): Promise<void>;

package/dist/mouse.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.moveToRegion = moveToRegion;
+exports.clickRegion = clickRegion;
+const nut_js_1 = require("@nut-tree-fork/nut-js");
+const config_1 = require("./config");
+const logger_1 = require("./logger");
+async function moveToRegion(region) {
+    const scaleFactor = config_1.visorConfig.scaleFactor;
+    const centerX = Math.floor((region.x + region.width / 2)
+        / scaleFactor);
+    const centerY = Math.floor((region.y + region.height / 2)
+        / scaleFactor);
+    (0, logger_1.log)("[MOUSE] Moving mouse to:", centerX, centerY);
+    await nut_js_1.mouse.move([
+        new nut_js_1.Point(centerX, centerY)
+    ]);
+}
+async function clickRegion(region) {
+    await moveToRegion(region);
+    await new Promise(r => setTimeout(r, 200));
+    await nut_js_1.mouse.click(nut_js_1.Button.LEFT);
+    (0, logger_1.log)("[MOUSE] Left click completed");
+}

package/dist/ocr.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Terminates OCR worker manually.
+ *
+ * Useful for cleanup after
+ * large automation sessions.
+ */
+export declare function terminateOCR(): Promise<void>;
+/**
+ * Extracts OCR text from either:
+ * - entire screen
+ * - specific screen region
+ *
+ * Uses:
+ * - screenshot-desktop
+ * - sharp preprocessing
+ * - Tesseract OCR
+ */
+export declare function extractTextFromRegion(region?: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}): Promise<any>;

package/dist/ocr.js

@@ -0,0 +1,83 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.terminateOCR = terminateOCR;
+exports.extractTextFromRegion = extractTextFromRegion;
+const screenshot_desktop_1 = __importDefault(require("screenshot-desktop"));
+const sharp_1 = __importDefault(require("sharp"));
+const tesseract_js_1 = require("tesseract.js");
+const config_1 = require("./config");
+const logger_1 = require("./logger");
+let worker = null;
+/**
+ * Returns shared OCR worker.
+ *
+ * Worker is initialized only once
+ * and reused across all OCR calls.
+ */
+async function getWorker() {
+    if (!worker) {
+        (0, logger_1.log)("[OCR] Initializing worker...");
+        worker = await (0, tesseract_js_1.createWorker)("eng");
+        (0, logger_1.log)("[OCR] Worker initialized");
+    }
+    return worker;
+}
+/**
+ * Terminates OCR worker manually.
+ *
+ * Useful for cleanup after
+ * large automation sessions.
+ */
+async function terminateOCR() {
+    if (worker) {
+        (0, logger_1.log)("[OCR] Terminating worker");
+        await worker.terminate();
+        worker = null;
+    }
+}
+/**
+ * Extracts OCR text from either:
+ * - entire screen
+ * - specific screen region
+ *
+ * Uses:
+ * - screenshot-desktop
+ * - sharp preprocessing
+ * - Tesseract OCR
+ */
+async function extractTextFromRegion(region) {
+    const buffer = await (0, screenshot_desktop_1.default)({
+        format: "png"
+    });
+    let image = (0, sharp_1.default)(buffer);
+    if (region) {
+        image = image.extract({
+            left: Math.floor(region.x *
+                config_1.visorConfig.scaleFactor),
+            top: Math.floor(region.y *
+                config_1.visorConfig.scaleFactor),
+            width: Math.floor(region.width *
+                config_1.visorConfig.scaleFactor),
+            height: Math.floor(region.height *
+                config_1.visorConfig.scaleFactor)
+        });
+    }
+    const processed = await image
+        .grayscale()
+        .normalize()
+        .sharpen()
+        .png()
+        .toBuffer();
+    const worker = await getWorker();
+    const result = await worker.recognize(processed, {}, {
+        blocks: true,
+        hocr: true,
+        tsv: true
+    });
+    (0, logger_1.log)("[OCR] Extracted Text:");
+    (0, logger_1.log)(result.data.text);
+    return result.data;
+}

package/dist/path.d.ts

@@ -0,0 +1 @@
+export declare function resolveImagePath(image: string): string;

package/dist/path.js

@@ -0,0 +1,15 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveImagePath = resolveImagePath;
+const path_1 = __importDefault(require("path"));
+const config_1 = require("./config");
+function resolveImagePath(image) {
+    if (image.includes("/") ||
+        image.includes("\\")) {
+        return image;
+    }
+    return path_1.default.join(config_1.visorConfig.imagePath, image);
+}

package/dist/screen.d.ts

@@ -0,0 +1,2 @@
+export declare function captureScreen(): Promise<any>;
+export declare function saveScreenshot(path: string): Promise<void>;

package/dist/screen.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureScreen = captureScreen;
+exports.saveScreenshot = saveScreenshot;
+const screenshot_desktop_1 = __importDefault(require("screenshot-desktop"));
+const pngjs_1 = require("pngjs");
+const fs_1 = __importDefault(require("fs"));
+const cv = require("@techstark/opencv-js");
+async function captureScreen() {
+    const buffer = await (0, screenshot_desktop_1.default)({
+        format: "png"
+    });
+    const png = pngjs_1.PNG.sync.read(buffer);
+    return cv.matFromImageData({
+        data: png.data,
+        width: png.width,
+        height: png.height
+    });
+}
+async function saveScreenshot(path) {
+    const img = await (0, screenshot_desktop_1.default)({
+        format: "png"
+    });
+    fs_1.default.writeFileSync(path, img);
+}

package/dist/src/index.d.ts

@@ -0,0 +1,6 @@
+export declare class Visor {
+    click(image: string, confidence?: number): Promise<void>;
+    exists(image: string, confidence?: number): Promise<boolean>;
+    wait(image: string, timeout?: number): Promise<boolean>;
+}
+export declare const visor: Visor;

package/dist/src/index.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.visor = exports.Visor = void 0;
+const screen_1 = require("./screen");
+const matcher_1 = require("./matcher");
+const mouse_1 = require("./mouse");
+class Visor {
+    async click(image, confidence = 0.8) {
+        console.log("Capturing screen...");
+        const screen = await (0, screen_1.captureScreen)();
+        console.log("Loading template...");
+        const template = (0, matcher_1.loadTemplate)(`src/images/${image}`);
+        console.log("Matching image...");
+        const region = (0, matcher_1.matchTemplate)(screen, template, confidence);
+        if (!region) {
+            console.log("Image not found");
+            throw new Error("Image not found");
+        }
+        console.log("Match found:", region);
+        await (0, mouse_1.clickRegion)(region);
+        console.log("Click completed");
+    }
+    async exists(image, confidence = 0.8) {
+        const screen = await (0, screen_1.captureScreen)();
+        const template = (0, matcher_1.loadTemplate)(`src/images/${image}`);
+        return (0, matcher_1.matchTemplate)(screen, template, confidence) !== null;
+    }
+    async wait(image, timeout = 5000) {
+        const start = Date.now();
+        while (Date.now() - start < timeout) {
+            if (await this.exists(image))
+                return true;
+            await new Promise(r => setTimeout(r, 300));
+        }
+        throw new Error("Timeout waiting for image");
+    }
+}
+exports.Visor = Visor;
+exports.visor = new Visor();

package/dist/src/matcher.d.ts

@@ -0,0 +1,3 @@
+import { Region } from "./types";
+export declare function loadTemplate(path: string): any;
+export declare function matchTemplate(screen: any, template: any, confidence?: number): Region | null;

package/dist/src/matcher.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadTemplate = loadTemplate;
+exports.matchTemplate = matchTemplate;
+const fs_1 = __importDefault(require("fs"));
+const pngjs_1 = require("pngjs");
+const cv = require("opencv.js");
+function loadTemplate(path) {
+    const buffer = fs_1.default.readFileSync(path);
+    const png = pngjs_1.PNG.sync.read(buffer);
+    return cv.matFromImageData({
+        data: png.data,
+        width: png.width,
+        height: png.height
+    });
+}
+function matchTemplate(screen, template, confidence = 0.8) {
+    const result = new cv.Mat();
+    cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
+    const { maxLoc, maxVal } = cv.minMaxLoc(result);
+    if (maxVal < confidence)
+        return null;
+    return {
+        x: maxLoc.x,
+        y: maxLoc.y,
+        width: template.cols,
+        height: template.rows,
+        confidence: maxVal
+    };
+}

package/dist/src/mouse.d.ts

@@ -0,0 +1,2 @@
+import { Region } from "./types";
+export declare function clickRegion(region: Region): Promise<void>;

package/dist/src/mouse.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.clickRegion = clickRegion;
+const nut_js_1 = require("@nut-tree-fork/nut-js");
+async function clickRegion(region) {
+    const scaleFactor = 1.5;
+    const centerX = Math.floor((region.x + region.width / 2) / scaleFactor);
+    const centerY = Math.floor((region.y + region.height / 2) / scaleFactor);
+    console.log("Moving mouse to:", centerX, centerY);
+    await nut_js_1.mouse.move([
+        new nut_js_1.Point(centerX, centerY)
+    ]);
+    console.log("Mouse moved");
+    await new Promise(r => setTimeout(r, 2000));
+    console.log("Clicking now");
+    await nut_js_1.mouse.click(nut_js_1.Button.LEFT);
+    console.log("Click done");
+}

package/dist/src/screen.d.ts

@@ -0,0 +1 @@
+export declare function captureScreen(): Promise<any>;

package/dist/src/screen.js

@@ -0,0 +1,20 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureScreen = captureScreen;
+const screenshot_desktop_1 = __importDefault(require("screenshot-desktop"));
+const pngjs_1 = require("pngjs");
+const cv = require("opencv.js");
+async function captureScreen() {
+    const buffer = await (0, screenshot_desktop_1.default)({
+        format: "png"
+    });
+    const png = pngjs_1.PNG.sync.read(buffer);
+    return cv.matFromImageData({
+        data: png.data,
+        width: png.width,
+        height: png.height
+    });
+}

package/dist/src/types.d.ts

@@ -0,0 +1,7 @@
+export interface Region {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    confidence: number;
+}

package/dist/src/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/text.d.ts

@@ -0,0 +1,13 @@
+import { Region } from "./types";
+export declare function findText(text: string, region?: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}): Promise<Region | null>;
+export declare function existsText(text: string, region?: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}): Promise<boolean>;

package/dist/text.js

@@ -0,0 +1,51 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findText = findText;
+exports.existsText = existsText;
+const ocr_1 = require("./ocr");
+async function findText(text, region) {
+    const result = await (0, ocr_1.extractTextFromRegion)(region);
+    const tsv = result.tsv;
+    if (!tsv) {
+        return null;
+    }
+    const lines = tsv.split("\n");
+    const words = [];
+    for (const line of lines) {
+        const cols = line.split("\t");
+        if (cols[0] !== "5")
+            continue;
+        const wordText = cols[11];
+        if (!wordText)
+            continue;
+        const confidence = Number(cols[10]);
+        if (confidence < 40)
+            continue;
+        words.push({
+            text: wordText,
+            x: Number(cols[6]),
+            y: Number(cols[7]),
+            width: Number(cols[8]),
+            height: Number(cols[9]),
+            confidence
+        });
+    }
+    //console.log(words);
+    const match = words.find(w => w.text
+        .toLowerCase()
+        .includes(text.toLowerCase()));
+    if (!match) {
+        return null;
+    }
+    return {
+        x: match.x,
+        y: match.y,
+        width: match.width,
+        height: match.height,
+        confidence: match.confidence
+    };
+}
+async function existsText(text, region) {
+    const match = await findText(text, region);
+    return match !== null;
+}

package/dist/types.d.ts

@@ -0,0 +1,7 @@
+export interface Region {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    confidence: number;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@aspiresys/visor",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "start": "ts-node src/test.ts"
+  },
+  "dependencies": {
+    "@nut-tree-fork/nut-js": "^4.1.0",
+    "@techstark/opencv-js": "^4.12.0-release.1",
+    "pngjs": "^7.0.0",
+    "screenshot-desktop": "^1.15.1",
+    "sharp": "^0.34.5",
+    "tesseract.js": "^7.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "@types/pngjs": "^6.0.5",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.9.0"
+  },
+  "description": "Desktop visual automation framework using OpenCV, OCR, and desktop interaction APIs.",
+
+  "keywords": [
+    "automation",
+    "desktop-automation",
+    "opencv",
+    "ocr",
+    "playwright",
+    "testing",
+    "computer-vision"
+  ],
+
+  "license": "MIT",
+
+  "files": [
+    "dist"
+  ]
+}