@aspiresys/visor 1.1.5 → 1.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.js +4 -4
- package/dist/display.d.ts +1 -0
- package/dist/display.js +16 -0
- package/dist/index.d.ts +785 -783
- package/dist/index.js +812 -803
- package/dist/matcher.js +16 -9
- package/dist/mouse.js +2 -4
- package/dist/ocr.js +12 -13
- package/dist/screen.js +1 -1
- package/dist/text.js +17 -11
- package/package.json +1 -1
- package/readme.md +171 -203
package/dist/matcher.js
CHANGED
|
@@ -9,17 +9,25 @@ exports.findAllMatches = findAllMatches;
|
|
|
9
9
|
const fs_1 = __importDefault(require("fs"));
|
|
10
10
|
const pngjs_1 = require("pngjs");
|
|
11
11
|
const cv = require("@techstark/opencv-js");
|
|
12
|
+
const templateCache = new Map();
|
|
12
13
|
const sharp_1 = __importDefault(require("sharp"));
|
|
13
14
|
const logger_1 = require("./logger");
|
|
14
15
|
async function loadTemplate(path) {
|
|
16
|
+
if (templateCache.has(path)) {
|
|
17
|
+
(0, logger_1.log)(`[CACHE] Template hit: ${path}`);
|
|
18
|
+
return templateCache.get(path).clone();
|
|
19
|
+
}
|
|
20
|
+
(0, logger_1.log)(`[CACHE] Loading template: ${path}`);
|
|
15
21
|
const buffer = fs_1.default.readFileSync(path);
|
|
16
22
|
const processed = await (0, sharp_1.default)(buffer).normalize().sharpen().png().toBuffer();
|
|
17
23
|
const png = pngjs_1.PNG.sync.read(processed);
|
|
18
|
-
|
|
24
|
+
const template = cv.matFromImageData({
|
|
19
25
|
data: png.data,
|
|
20
26
|
width: png.width,
|
|
21
|
-
height: png.height
|
|
27
|
+
height: png.height,
|
|
22
28
|
});
|
|
29
|
+
templateCache.set(path, template);
|
|
30
|
+
return template.clone();
|
|
23
31
|
}
|
|
24
32
|
function resizeTemplate(template, scale) {
|
|
25
33
|
const resized = new cv.Mat();
|
|
@@ -33,7 +41,8 @@ function matchTemplate(screen, template, confidence = 0.8) {
|
|
|
33
41
|
let bestMatch = null;
|
|
34
42
|
for (const scale of scales) {
|
|
35
43
|
const resizedTemplate = resizeTemplate(template, scale);
|
|
36
|
-
if (resizedTemplate.cols > screen.cols ||
|
|
44
|
+
if (resizedTemplate.cols > screen.cols ||
|
|
45
|
+
resizedTemplate.rows > screen.rows) {
|
|
37
46
|
resizedTemplate.delete();
|
|
38
47
|
continue;
|
|
39
48
|
}
|
|
@@ -42,15 +51,13 @@ function matchTemplate(screen, template, confidence = 0.8) {
|
|
|
42
51
|
const { maxLoc, maxVal } = cv.minMaxLoc(result);
|
|
43
52
|
(0, logger_1.log)(`[MATCH] Scale ${scale} -> ${maxVal}`);
|
|
44
53
|
result.delete();
|
|
45
|
-
if (maxVal >= confidence &&
|
|
46
|
-
(!bestMatch ||
|
|
47
|
-
maxVal > bestMatch.confidence)) {
|
|
54
|
+
if (maxVal >= confidence && (!bestMatch || maxVal > bestMatch.confidence)) {
|
|
48
55
|
bestMatch = {
|
|
49
56
|
x: maxLoc.x,
|
|
50
57
|
y: maxLoc.y,
|
|
51
58
|
width: resizedTemplate.cols,
|
|
52
59
|
height: resizedTemplate.rows,
|
|
53
|
-
confidence: maxVal
|
|
60
|
+
confidence: maxVal,
|
|
54
61
|
};
|
|
55
62
|
}
|
|
56
63
|
resizedTemplate.delete();
|
|
@@ -70,7 +77,7 @@ function findAllMatches(screen, template, confidence = 0.8) {
|
|
|
70
77
|
y,
|
|
71
78
|
width: template.cols,
|
|
72
79
|
height: template.rows,
|
|
73
|
-
confidence: score
|
|
80
|
+
confidence: score,
|
|
74
81
|
});
|
|
75
82
|
}
|
|
76
83
|
}
|
|
@@ -78,7 +85,7 @@ function findAllMatches(screen, template, confidence = 0.8) {
|
|
|
78
85
|
matches.sort((a, b) => b.confidence - a.confidence);
|
|
79
86
|
const filtered = [];
|
|
80
87
|
for (const match of matches) {
|
|
81
|
-
const tooClose = filtered.some(existing => {
|
|
88
|
+
const tooClose = filtered.some((existing) => {
|
|
82
89
|
const dx = Math.abs(existing.x - match.x);
|
|
83
90
|
const dy = Math.abs(existing.y - match.y);
|
|
84
91
|
return dx < 10 && dy < 10;
|
package/dist/mouse.js
CHANGED
|
@@ -10,13 +10,11 @@ async function moveToRegion(region) {
|
|
|
10
10
|
const centerX = Math.floor((region.x + region.width / 2) / scaleFactor);
|
|
11
11
|
const centerY = Math.floor((region.y + region.height / 2) / scaleFactor);
|
|
12
12
|
(0, logger_1.log)("[MOUSE] Moving mouse to:", centerX, centerY);
|
|
13
|
-
await nut_js_1.mouse.move([
|
|
14
|
-
new nut_js_1.Point(centerX, centerY)
|
|
15
|
-
]);
|
|
13
|
+
await nut_js_1.mouse.move([new nut_js_1.Point(centerX, centerY)]);
|
|
16
14
|
}
|
|
17
15
|
async function clickRegion(region) {
|
|
18
16
|
await moveToRegion(region);
|
|
19
|
-
await new Promise(r => setTimeout(r, 200));
|
|
17
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
20
18
|
await nut_js_1.mouse.click(nut_js_1.Button.LEFT);
|
|
21
19
|
(0, logger_1.log)("[MOUSE] Left click completed");
|
|
22
20
|
}
|
package/dist/ocr.js
CHANGED
|
@@ -49,23 +49,22 @@ async function terminateOCR() {
|
|
|
49
49
|
* - Tesseract OCR
|
|
50
50
|
*/
|
|
51
51
|
async function extractTextFromRegion(region) {
|
|
52
|
-
const buffer = await (0, screenshot_desktop_1.default)({
|
|
53
|
-
format: "png"
|
|
54
|
-
});
|
|
52
|
+
const buffer = await (0, screenshot_desktop_1.default)({ format: "png" });
|
|
55
53
|
let image = (0, sharp_1.default)(buffer);
|
|
56
54
|
if (region) {
|
|
57
55
|
image = image.extract({
|
|
58
|
-
left: Math.floor(region.x *
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
width: Math.floor(region.width *
|
|
63
|
-
config_1.visorConfig.scaleFactor),
|
|
64
|
-
height: Math.floor(region.height *
|
|
65
|
-
config_1.visorConfig.scaleFactor)
|
|
56
|
+
left: Math.floor(region.x * config_1.visorConfig.scaleFactor),
|
|
57
|
+
top: Math.floor(region.y * config_1.visorConfig.scaleFactor),
|
|
58
|
+
width: Math.floor(region.width * config_1.visorConfig.scaleFactor),
|
|
59
|
+
height: Math.floor(region.height * config_1.visorConfig.scaleFactor),
|
|
66
60
|
});
|
|
67
61
|
}
|
|
68
|
-
const processed = await image
|
|
62
|
+
const processed = await image
|
|
63
|
+
.grayscale()
|
|
64
|
+
.normalize()
|
|
65
|
+
.sharpen()
|
|
66
|
+
.png()
|
|
67
|
+
.toBuffer();
|
|
69
68
|
const worker = await getWorker();
|
|
70
69
|
const result = await worker.recognize(processed, {}, {
|
|
71
70
|
blocks: true,
|
|
@@ -73,7 +72,7 @@ async function extractTextFromRegion(region) {
|
|
|
73
72
|
tsv: true,
|
|
74
73
|
tessedit_pageseg_mode: "6",
|
|
75
74
|
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .:-/₹$€-,",
|
|
76
|
-
user_defined_dpi: "300"
|
|
75
|
+
user_defined_dpi: "300",
|
|
77
76
|
});
|
|
78
77
|
(0, logger_1.log)("[OCR] Extracted Text:");
|
|
79
78
|
(0, logger_1.log)(result.data.text);
|
package/dist/screen.js
CHANGED
package/dist/text.js
CHANGED
|
@@ -37,7 +37,7 @@ async function findText(text, index = 0, region) {
|
|
|
37
37
|
y: Number(cols[7]),
|
|
38
38
|
width: Number(cols[8]),
|
|
39
39
|
height: Number(cols[9]),
|
|
40
|
-
confidence
|
|
40
|
+
confidence,
|
|
41
41
|
});
|
|
42
42
|
}
|
|
43
43
|
const matches = [];
|
|
@@ -46,26 +46,32 @@ async function findText(text, index = 0, region) {
|
|
|
46
46
|
words.sort((a, b) => a.x - b.x);
|
|
47
47
|
const searchWords = text.toLowerCase().split(/\s+/);
|
|
48
48
|
//const combinedText = words.map(w => w.text).join(" ");
|
|
49
|
-
const combinedText = words
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
const combinedText = words
|
|
50
|
+
.map((w) => w.text)
|
|
51
|
+
.join(" ")
|
|
52
|
+
.replace(/\s+/g, " ")
|
|
53
|
+
.trim();
|
|
54
|
+
if (combinedText
|
|
55
|
+
.toLowerCase()
|
|
56
|
+
.replace(/[^a-z0-9 ]/gi, "")
|
|
57
|
+
.includes(text.toLowerCase().replace(/[^a-z0-9 ]/gi, ""))) {
|
|
58
|
+
const matchedWords = words.filter((w) => searchWords.includes(w.text.toLowerCase().replace(/[^a-z0-9]/gi, "")));
|
|
52
59
|
const regionWords = matchedWords.length > 0 ? matchedWords : words;
|
|
53
|
-
const minX = Math.min(...regionWords.map(w => w.x));
|
|
54
|
-
const minY = Math.min(...regionWords.map(w => w.y));
|
|
55
|
-
const maxX = Math.max(...regionWords.map(w => w.x + w.width));
|
|
56
|
-
const maxY = Math.max(...regionWords.map(w => w.y + w.height));
|
|
60
|
+
const minX = Math.min(...regionWords.map((w) => w.x));
|
|
61
|
+
const minY = Math.min(...regionWords.map((w) => w.y));
|
|
62
|
+
const maxX = Math.max(...regionWords.map((w) => w.x + w.width));
|
|
63
|
+
const maxY = Math.max(...regionWords.map((w) => w.y + w.height));
|
|
57
64
|
const avgConfidence = words.reduce((sum, w) => sum + w.confidence, 0) / words.length;
|
|
58
65
|
matches.push({
|
|
59
66
|
x: minX,
|
|
60
67
|
y: minY,
|
|
61
68
|
width: maxX - minX,
|
|
62
69
|
height: maxY - minY,
|
|
63
|
-
confidence: avgConfidence
|
|
70
|
+
confidence: avgConfidence,
|
|
64
71
|
});
|
|
65
72
|
}
|
|
66
73
|
}
|
|
67
|
-
matches.sort((a, b) => a.y - b.y ||
|
|
68
|
-
a.x - b.x);
|
|
74
|
+
matches.sort((a, b) => a.y - b.y || a.x - b.x);
|
|
69
75
|
return matches[index] || null;
|
|
70
76
|
}
|
|
71
77
|
async function existsText(text, region) {
|