@aspiresys/visor 1.3.4 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.js +2 -2
- package/dist/config.js +2 -2
- package/dist/dialog.js +4 -4
- package/dist/display.js +21 -7
- package/dist/index.d.ts +156 -139
- package/dist/index.js +197 -163
- package/dist/inspector/index.js +10 -8
- package/dist/matcher.d.ts +2 -2
- package/dist/matcher.js +50 -25
- package/dist/mouse.d.ts +1 -1
- package/dist/mouse.js +4 -4
- package/dist/ocr.d.ts +2 -1
- package/dist/ocr.js +15 -15
- package/dist/path.js +1 -1
- package/dist/region.js +3 -4
- package/dist/screen.js +5 -5
- package/dist/src/index.js +15 -16
- package/dist/src/matcher.d.ts +1 -1
- package/dist/src/matcher.js +13 -12
- package/dist/src/mouse.d.ts +1 -1
- package/dist/src/mouse.js +9 -11
- package/dist/src/screen.js +12 -10
- package/dist/src/types.js +2 -2
- package/dist/templateMetadata.d.ts +1 -0
- package/dist/templateMetadata.js +20 -0
- package/dist/text.d.ts +1 -1
- package/dist/text.js +8 -8
- package/dist/types.d.ts +249 -159
- package/dist/types.js +330 -170
- package/dist/version.d.ts +1 -0
- package/dist/version.js +18 -0
- package/inspector/index.html +82 -55
- package/inspector/main.js +135 -189
- package/inspector/src/preload.js +10 -15
- package/inspector/src/renderer.js +177 -235
- package/inspector/start.js +8 -16
- package/inspector/styles.css +456 -458
- package/package.json +57 -51
- package/readme.md +790 -799
package/dist/matcher.js
CHANGED
|
@@ -12,10 +12,11 @@ exports.matchInspectorTemplate = matchInspectorTemplate;
|
|
|
12
12
|
exports.cropMat = cropMat;
|
|
13
13
|
const fs_1 = __importDefault(require("fs"));
|
|
14
14
|
const pngjs_1 = require("pngjs");
|
|
15
|
-
const
|
|
15
|
+
const opencv_js_1 = __importDefault(require("@techstark/opencv-js"));
|
|
16
16
|
const templateCache = new Map();
|
|
17
17
|
const types_1 = require("./types");
|
|
18
18
|
const logger_1 = require("./logger");
|
|
19
|
+
const config_1 = require("./config");
|
|
19
20
|
async function loadTemplate(path) {
|
|
20
21
|
if (templateCache.has(path)) {
|
|
21
22
|
(0, logger_1.log)(`[CACHE] Template hit: ${path}`);
|
|
@@ -24,7 +25,7 @@ async function loadTemplate(path) {
|
|
|
24
25
|
(0, logger_1.log)(`[CACHE] Loading template: ${path}`);
|
|
25
26
|
const buffer = fs_1.default.readFileSync(path);
|
|
26
27
|
const png = pngjs_1.PNG.sync.read(buffer);
|
|
27
|
-
const template =
|
|
28
|
+
const template = opencv_js_1.default.matFromImageData({
|
|
28
29
|
data: png.data,
|
|
29
30
|
width: png.width,
|
|
30
31
|
height: png.height,
|
|
@@ -36,33 +37,58 @@ async function loadScreen(path) {
|
|
|
36
37
|
(0, logger_1.log)(`[SCREEN] Loading: ${path}`);
|
|
37
38
|
const buffer = fs_1.default.readFileSync(path);
|
|
38
39
|
const png = pngjs_1.PNG.sync.read(buffer);
|
|
39
|
-
return
|
|
40
|
+
return opencv_js_1.default.matFromImageData({
|
|
40
41
|
data: png.data,
|
|
41
42
|
width: png.width,
|
|
42
43
|
height: png.height,
|
|
43
44
|
});
|
|
44
45
|
}
|
|
45
46
|
function resizeTemplate(template, scale) {
|
|
46
|
-
const resized = new
|
|
47
|
+
const resized = new opencv_js_1.default.Mat();
|
|
47
48
|
const newWidth = Math.max(1, Math.floor(template.cols * scale));
|
|
48
49
|
const newHeight = Math.max(1, Math.floor(template.rows * scale));
|
|
49
|
-
|
|
50
|
+
opencv_js_1.default.resize(template, resized, new opencv_js_1.default.Size(newWidth, newHeight), 0, 0, scale < 1 ? opencv_js_1.default.INTER_AREA : opencv_js_1.default.INTER_CUBIC);
|
|
50
51
|
return resized;
|
|
51
52
|
}
|
|
52
|
-
function matchTemplate(screen, template, confidence = 0.8) {
|
|
53
|
+
function matchTemplate(screen, template, confidence = 0.8, metadata) {
|
|
54
|
+
(0, logger_1.log)('Screen:', screen.cols, 'x', screen.rows);
|
|
55
|
+
(0, logger_1.log)('Template:', template.cols, 'x', template.rows);
|
|
53
56
|
const start = Date.now();
|
|
54
|
-
const scales =
|
|
57
|
+
const scales = (() => {
|
|
58
|
+
if (metadata) {
|
|
59
|
+
try {
|
|
60
|
+
const widthRatio = metadata.currentResolution.width / metadata.capturedResolution.width;
|
|
61
|
+
const heightRatio = metadata.currentResolution.height / metadata.capturedResolution.height;
|
|
62
|
+
const resolutionRatio = (widthRatio + heightRatio) / 2;
|
|
63
|
+
const dpiRatio = config_1.visorConfig.scaleFactor / metadata.capturedScaleFactor;
|
|
64
|
+
const predictedScale = resolutionRatio * dpiRatio;
|
|
65
|
+
(0, logger_1.log)(`[MATCH] Predicted scale: ${predictedScale.toFixed(3)}`);
|
|
66
|
+
return [
|
|
67
|
+
predictedScale,
|
|
68
|
+
predictedScale * 0.95,
|
|
69
|
+
predictedScale * 1.05,
|
|
70
|
+
predictedScale * 0.9,
|
|
71
|
+
predictedScale * 1.1,
|
|
72
|
+
];
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
(0, logger_1.log)('[MATCH] Error calculating predicted scale:', error);
|
|
76
|
+
(0, logger_1.log)('[MATCH] Falling back to default scales');
|
|
77
|
+
return [1.0, 1.25, 1.5, 1.75, 2.0, 0.75, 0.5];
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return [1.0, 1.25, 1.5, 1.75, 2.0, 0.75, 0.5];
|
|
81
|
+
})();
|
|
55
82
|
let bestMatch = null;
|
|
56
83
|
for (const scale of scales) {
|
|
57
84
|
const resizedTemplate = resizeTemplate(template, scale);
|
|
58
|
-
if (resizedTemplate.cols > screen.cols ||
|
|
59
|
-
resizedTemplate.rows > screen.rows) {
|
|
85
|
+
if (resizedTemplate.cols > screen.cols || resizedTemplate.rows > screen.rows) {
|
|
60
86
|
resizedTemplate.delete();
|
|
61
87
|
continue;
|
|
62
88
|
}
|
|
63
|
-
const result = new
|
|
64
|
-
|
|
65
|
-
const { maxLoc, maxVal } =
|
|
89
|
+
const result = new opencv_js_1.default.Mat();
|
|
90
|
+
opencv_js_1.default.matchTemplate(screen, resizedTemplate, result, opencv_js_1.default.TM_CCOEFF_NORMED);
|
|
91
|
+
const { maxLoc, maxVal } = opencv_js_1.default.minMaxLoc(result);
|
|
66
92
|
(0, logger_1.log)(`[MATCH] Scale ${scale} -> ${maxVal}`);
|
|
67
93
|
result.delete();
|
|
68
94
|
if (maxVal >= confidence && (!bestMatch || maxVal > bestMatch.confidence)) {
|
|
@@ -79,8 +105,8 @@ function matchTemplate(screen, template, confidence = 0.8) {
|
|
|
79
105
|
return bestMatch;
|
|
80
106
|
}
|
|
81
107
|
function findAllMatches(screen, template, confidence = 0.8) {
|
|
82
|
-
const result = new
|
|
83
|
-
|
|
108
|
+
const result = new opencv_js_1.default.Mat();
|
|
109
|
+
opencv_js_1.default.matchTemplate(screen, template, result, opencv_js_1.default.TM_CCOEFF_NORMED);
|
|
84
110
|
const matches = [];
|
|
85
111
|
for (let y = 0; y < result.rows; y++) {
|
|
86
112
|
for (let x = 0; x < result.cols; x++) {
|
|
@@ -106,8 +132,8 @@ function findAllMatches(screen, template, confidence = 0.8) {
|
|
|
106
132
|
return filtered;
|
|
107
133
|
}
|
|
108
134
|
function findAllInspectorMatches(screen, template, confidence = 0.8) {
|
|
109
|
-
const result = new
|
|
110
|
-
|
|
135
|
+
const result = new opencv_js_1.default.Mat();
|
|
136
|
+
opencv_js_1.default.matchTemplate(screen, template, result, opencv_js_1.default.TM_CCOEFF_NORMED);
|
|
111
137
|
const matches = [];
|
|
112
138
|
for (let y = 0; y < result.rows; y++) {
|
|
113
139
|
for (let x = 0; x < result.cols; x++) {
|
|
@@ -145,14 +171,13 @@ function matchInspectorTemplate(screen, template, confidence = 0.8, multiScale =
|
|
|
145
171
|
let bestMatch = null;
|
|
146
172
|
for (const scale of scales) {
|
|
147
173
|
const resizedTemplate = resizeTemplate(template, scale);
|
|
148
|
-
if (resizedTemplate.cols > screen.cols ||
|
|
149
|
-
resizedTemplate.rows > screen.rows) {
|
|
174
|
+
if (resizedTemplate.cols > screen.cols || resizedTemplate.rows > screen.rows) {
|
|
150
175
|
resizedTemplate.delete();
|
|
151
176
|
continue;
|
|
152
177
|
}
|
|
153
|
-
const result = new
|
|
154
|
-
|
|
155
|
-
const { maxLoc, maxVal } =
|
|
178
|
+
const result = new opencv_js_1.default.Mat();
|
|
179
|
+
opencv_js_1.default.matchTemplate(screen, resizedTemplate, result, opencv_js_1.default.TM_CCOEFF_NORMED);
|
|
180
|
+
const { maxLoc, maxVal } = opencv_js_1.default.minMaxLoc(result);
|
|
156
181
|
(0, logger_1.log)(`[MATCH] Scale ${scale} -> ${maxVal}`);
|
|
157
182
|
result.delete();
|
|
158
183
|
if (maxVal >= confidence && (!bestMatch || maxVal > bestMatch.confidence)) {
|
|
@@ -170,9 +195,9 @@ function matchInspectorTemplate(screen, template, confidence = 0.8, multiScale =
|
|
|
170
195
|
}
|
|
171
196
|
else {
|
|
172
197
|
const start = Date.now();
|
|
173
|
-
const result = new
|
|
174
|
-
|
|
175
|
-
const { maxLoc, maxVal } =
|
|
198
|
+
const result = new opencv_js_1.default.Mat();
|
|
199
|
+
opencv_js_1.default.matchTemplate(screen, template, result, opencv_js_1.default.TM_CCOEFF_NORMED);
|
|
200
|
+
const { maxLoc, maxVal } = opencv_js_1.default.minMaxLoc(result);
|
|
176
201
|
result.delete();
|
|
177
202
|
if (maxVal >= confidence) {
|
|
178
203
|
return new types_1.Region(maxLoc.x, maxLoc.y, template.cols, template.rows, maxVal);
|
|
@@ -186,7 +211,7 @@ function cropMat(screen, region) {
|
|
|
186
211
|
const y = Math.max(0, Math.floor(region.y));
|
|
187
212
|
const width = Math.min(Math.floor(region.width), screen.cols - x);
|
|
188
213
|
const height = Math.min(Math.floor(region.height), screen.rows - y);
|
|
189
|
-
const rect = new
|
|
214
|
+
const rect = new opencv_js_1.default.Rect(x, y, width, height);
|
|
190
215
|
const croppedMat = screen.roi(rect);
|
|
191
216
|
const isolatedMat = croppedMat.clone();
|
|
192
217
|
croppedMat.delete();
|
package/dist/mouse.d.ts
CHANGED
package/dist/mouse.js
CHANGED
|
@@ -7,21 +7,21 @@ const config_1 = require("./config");
|
|
|
7
7
|
const logger_1 = require("./logger");
|
|
8
8
|
async function moveToRegion(region, offset = {
|
|
9
9
|
x: 0,
|
|
10
|
-
y: 0
|
|
10
|
+
y: 0,
|
|
11
11
|
}) {
|
|
12
12
|
const scaleFactor = config_1.visorConfig.scaleFactor;
|
|
13
13
|
const center = region.center();
|
|
14
14
|
const centerX = Math.floor((center.x + offset.x) / scaleFactor);
|
|
15
15
|
const centerY = Math.floor((center.y + offset.y) / scaleFactor);
|
|
16
|
-
(0, logger_1.log)(
|
|
16
|
+
(0, logger_1.log)('[MOUSE] Moving mouse to:', centerX, centerY);
|
|
17
17
|
await nut_js_1.mouse.move([new nut_js_1.Point(centerX, centerY)]);
|
|
18
18
|
}
|
|
19
19
|
async function clickRegion(region, offset = {
|
|
20
20
|
x: 0,
|
|
21
|
-
y: 0
|
|
21
|
+
y: 0,
|
|
22
22
|
}) {
|
|
23
23
|
await moveToRegion(region, offset);
|
|
24
24
|
await new Promise((r) => setTimeout(r, 200));
|
|
25
25
|
await nut_js_1.mouse.click(nut_js_1.Button.LEFT);
|
|
26
|
-
(0, logger_1.log)(
|
|
26
|
+
(0, logger_1.log)('[MOUSE] Left click completed');
|
|
27
27
|
}
|
package/dist/ocr.d.ts
CHANGED
package/dist/ocr.js
CHANGED
|
@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.initializeOCR = initializeOCR;
|
|
6
7
|
exports.terminateOCR = terminateOCR;
|
|
7
8
|
exports.extractTextFromRegion = extractTextFromRegion;
|
|
8
9
|
const screenshot_desktop_1 = __importDefault(require("screenshot-desktop"));
|
|
@@ -19,12 +20,16 @@ let worker = null;
|
|
|
19
20
|
*/
|
|
20
21
|
async function getWorker() {
|
|
21
22
|
if (!worker) {
|
|
22
|
-
(0, logger_1.log)(
|
|
23
|
-
worker = await (0, tesseract_js_1.createWorker)(
|
|
24
|
-
(0, logger_1.log)(
|
|
23
|
+
(0, logger_1.log)('[OCR] Initializing worker...');
|
|
24
|
+
worker = await (0, tesseract_js_1.createWorker)('eng');
|
|
25
|
+
(0, logger_1.log)('[OCR] Worker initialized');
|
|
25
26
|
}
|
|
26
27
|
return worker;
|
|
27
28
|
}
|
|
29
|
+
async function initializeOCR() {
|
|
30
|
+
await getWorker();
|
|
31
|
+
(0, logger_1.log)('[OCR] Warmup complete');
|
|
32
|
+
}
|
|
28
33
|
/**
|
|
29
34
|
* Terminates OCR worker manually.
|
|
30
35
|
*
|
|
@@ -33,7 +38,7 @@ async function getWorker() {
|
|
|
33
38
|
*/
|
|
34
39
|
async function terminateOCR() {
|
|
35
40
|
if (worker) {
|
|
36
|
-
(0, logger_1.log)(
|
|
41
|
+
(0, logger_1.log)('[OCR] Terminating worker');
|
|
37
42
|
await worker.terminate();
|
|
38
43
|
worker = null;
|
|
39
44
|
}
|
|
@@ -49,7 +54,7 @@ async function terminateOCR() {
|
|
|
49
54
|
* - Tesseract OCR
|
|
50
55
|
*/
|
|
51
56
|
async function extractTextFromRegion(region) {
|
|
52
|
-
const buffer = await (0, screenshot_desktop_1.default)({ format:
|
|
57
|
+
const buffer = await (0, screenshot_desktop_1.default)({ format: 'png' });
|
|
53
58
|
let image = (0, sharp_1.default)(buffer);
|
|
54
59
|
if (region) {
|
|
55
60
|
image = image.extract({
|
|
@@ -59,22 +64,17 @@ async function extractTextFromRegion(region) {
|
|
|
59
64
|
height: Math.floor(region.height * config_1.visorConfig.scaleFactor),
|
|
60
65
|
});
|
|
61
66
|
}
|
|
62
|
-
const processed = await image
|
|
63
|
-
.grayscale()
|
|
64
|
-
.normalize()
|
|
65
|
-
.sharpen()
|
|
66
|
-
.png()
|
|
67
|
-
.toBuffer();
|
|
67
|
+
const processed = await image.grayscale().normalize().sharpen().png().toBuffer();
|
|
68
68
|
const worker = await getWorker();
|
|
69
69
|
const result = await worker.recognize(processed, {}, {
|
|
70
70
|
blocks: true,
|
|
71
71
|
hocr: true,
|
|
72
72
|
tsv: true,
|
|
73
|
-
tessedit_pageseg_mode:
|
|
74
|
-
tessedit_char_whitelist:
|
|
75
|
-
user_defined_dpi:
|
|
73
|
+
tessedit_pageseg_mode: '6',
|
|
74
|
+
tessedit_char_whitelist: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .:-/₹$€-,',
|
|
75
|
+
user_defined_dpi: '300',
|
|
76
76
|
});
|
|
77
|
-
(0, logger_1.log)(
|
|
77
|
+
(0, logger_1.log)('[OCR] Extracted Text:');
|
|
78
78
|
(0, logger_1.log)(result.data.text);
|
|
79
79
|
return result.data;
|
|
80
80
|
}
|
package/dist/path.js
CHANGED
|
@@ -7,7 +7,7 @@ exports.resolveImagePath = resolveImagePath;
|
|
|
7
7
|
const path_1 = __importDefault(require("path"));
|
|
8
8
|
const config_1 = require("./config");
|
|
9
9
|
function resolveImagePath(image) {
|
|
10
|
-
if (image.includes(
|
|
10
|
+
if (image.includes('/') || image.includes('\\')) {
|
|
11
11
|
return image;
|
|
12
12
|
}
|
|
13
13
|
return path_1.default.join(config_1.visorConfig.imagePath, image);
|
package/dist/region.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports,
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
3
3
|
exports.Region = void 0;
|
|
4
4
|
class Region {
|
|
5
5
|
constructor(x, y, width, height, confidence = 1) {
|
|
@@ -9,7 +9,6 @@ class Region {
|
|
|
9
9
|
this.height = height;
|
|
10
10
|
this.confidence = confidence;
|
|
11
11
|
}
|
|
12
|
-
center() {
|
|
13
|
-
}
|
|
12
|
+
center() {}
|
|
14
13
|
}
|
|
15
14
|
exports.Region = Region;
|
package/dist/screen.js
CHANGED
|
@@ -9,11 +9,11 @@ exports.getScreenshotBuffer = getScreenshotBuffer;
|
|
|
9
9
|
const screenshot_desktop_1 = __importDefault(require("screenshot-desktop"));
|
|
10
10
|
const pngjs_1 = require("pngjs");
|
|
11
11
|
const fs_1 = __importDefault(require("fs"));
|
|
12
|
-
const
|
|
12
|
+
const opencv_js_1 = __importDefault(require("@techstark/opencv-js"));
|
|
13
13
|
async function captureScreen() {
|
|
14
|
-
const buffer = await (0, screenshot_desktop_1.default)({ format:
|
|
14
|
+
const buffer = await (0, screenshot_desktop_1.default)({ format: 'png' });
|
|
15
15
|
const png = pngjs_1.PNG.sync.read(buffer);
|
|
16
|
-
const mat =
|
|
16
|
+
const mat = opencv_js_1.default.matFromImageData({
|
|
17
17
|
data: png.data,
|
|
18
18
|
width: png.width,
|
|
19
19
|
height: png.height,
|
|
@@ -21,11 +21,11 @@ async function captureScreen() {
|
|
|
21
21
|
return mat;
|
|
22
22
|
}
|
|
23
23
|
async function saveScreenshot(path) {
|
|
24
|
-
const img = await (0, screenshot_desktop_1.default)({ format:
|
|
24
|
+
const img = await (0, screenshot_desktop_1.default)({ format: 'png' });
|
|
25
25
|
fs_1.default.writeFileSync(path, img);
|
|
26
26
|
}
|
|
27
27
|
async function getScreenshotBuffer() {
|
|
28
28
|
return await (0, screenshot_desktop_1.default)({
|
|
29
|
-
format:
|
|
29
|
+
format: 'png',
|
|
30
30
|
});
|
|
31
31
|
}
|
package/dist/src/index.js
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports,
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
3
3
|
exports.visor = exports.Visor = void 0;
|
|
4
|
-
const screen_1 = require(
|
|
5
|
-
const matcher_1 = require(
|
|
6
|
-
const mouse_1 = require(
|
|
4
|
+
const screen_1 = require('./screen');
|
|
5
|
+
const matcher_1 = require('./matcher');
|
|
6
|
+
const mouse_1 = require('./mouse');
|
|
7
7
|
class Visor {
|
|
8
8
|
async click(image, confidence = 0.8) {
|
|
9
|
-
console.log(
|
|
9
|
+
console.log('Capturing screen...');
|
|
10
10
|
const screen = await (0, screen_1.captureScreen)();
|
|
11
|
-
console.log(
|
|
11
|
+
console.log('Loading template...');
|
|
12
12
|
const template = (0, matcher_1.loadTemplate)(`src/images/${image}`);
|
|
13
|
-
console.log(
|
|
13
|
+
console.log('Matching image...');
|
|
14
14
|
const region = (0, matcher_1.matchTemplate)(screen, template, confidence);
|
|
15
15
|
if (!region) {
|
|
16
|
-
console.log(
|
|
17
|
-
throw new Error(
|
|
16
|
+
console.log('Image not found');
|
|
17
|
+
throw new Error('Image not found');
|
|
18
18
|
}
|
|
19
|
-
console.log(
|
|
19
|
+
console.log('Match found:', region);
|
|
20
20
|
await (0, mouse_1.clickRegion)(region);
|
|
21
|
-
console.log(
|
|
21
|
+
console.log('Click completed');
|
|
22
22
|
}
|
|
23
23
|
async exists(image, confidence = 0.8) {
|
|
24
24
|
const screen = await (0, screen_1.captureScreen)();
|
|
@@ -28,11 +28,10 @@ class Visor {
|
|
|
28
28
|
async wait(image, timeout = 5000) {
|
|
29
29
|
const start = Date.now();
|
|
30
30
|
while (Date.now() - start < timeout) {
|
|
31
|
-
if (await this.exists(image))
|
|
32
|
-
|
|
33
|
-
await new Promise(r => setTimeout(r, 300));
|
|
31
|
+
if (await this.exists(image)) return true;
|
|
32
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
34
33
|
}
|
|
35
|
-
throw new Error(
|
|
34
|
+
throw new Error('Timeout waiting for image');
|
|
36
35
|
}
|
|
37
36
|
}
|
|
38
37
|
exports.Visor = Visor;
|
package/dist/src/matcher.d.ts
CHANGED
package/dist/src/matcher.js
CHANGED
|
@@ -1,33 +1,34 @@
|
|
|
1
|
-
|
|
2
|
-
var __importDefault =
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
'use strict';
|
|
2
|
+
var __importDefault =
|
|
3
|
+
(this && this.__importDefault) ||
|
|
4
|
+
function (mod) {
|
|
5
|
+
return mod && mod.__esModule ? mod : { default: mod };
|
|
6
|
+
};
|
|
7
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
6
8
|
exports.loadTemplate = loadTemplate;
|
|
7
9
|
exports.matchTemplate = matchTemplate;
|
|
8
|
-
const fs_1 = __importDefault(require(
|
|
9
|
-
const pngjs_1 = require(
|
|
10
|
-
const cv = require(
|
|
10
|
+
const fs_1 = __importDefault(require('fs'));
|
|
11
|
+
const pngjs_1 = require('pngjs');
|
|
12
|
+
const cv = require('opencv.js');
|
|
11
13
|
function loadTemplate(path) {
|
|
12
14
|
const buffer = fs_1.default.readFileSync(path);
|
|
13
15
|
const png = pngjs_1.PNG.sync.read(buffer);
|
|
14
16
|
return cv.matFromImageData({
|
|
15
17
|
data: png.data,
|
|
16
18
|
width: png.width,
|
|
17
|
-
height: png.height
|
|
19
|
+
height: png.height,
|
|
18
20
|
});
|
|
19
21
|
}
|
|
20
22
|
function matchTemplate(screen, template, confidence = 0.8) {
|
|
21
23
|
const result = new cv.Mat();
|
|
22
24
|
cv.matchTemplate(screen, template, result, cv.TM_CCOEFF_NORMED);
|
|
23
25
|
const { maxLoc, maxVal } = cv.minMaxLoc(result);
|
|
24
|
-
if (maxVal < confidence)
|
|
25
|
-
return null;
|
|
26
|
+
if (maxVal < confidence) return null;
|
|
26
27
|
return {
|
|
27
28
|
x: maxLoc.x,
|
|
28
29
|
y: maxLoc.y,
|
|
29
30
|
width: template.cols,
|
|
30
31
|
height: template.rows,
|
|
31
|
-
confidence: maxVal
|
|
32
|
+
confidence: maxVal,
|
|
32
33
|
};
|
|
33
34
|
}
|
package/dist/src/mouse.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { Region } from
|
|
1
|
+
import { Region } from './types';
|
|
2
2
|
export declare function clickRegion(region: Region): Promise<void>;
|
package/dist/src/mouse.js
CHANGED
|
@@ -1,18 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports,
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
3
3
|
exports.clickRegion = clickRegion;
|
|
4
|
-
const nut_js_1 = require(
|
|
4
|
+
const nut_js_1 = require('@nut-tree-fork/nut-js');
|
|
5
5
|
async function clickRegion(region) {
|
|
6
6
|
const scaleFactor = 1.5;
|
|
7
7
|
const centerX = Math.floor((region.x + region.width / 2) / scaleFactor);
|
|
8
8
|
const centerY = Math.floor((region.y + region.height / 2) / scaleFactor);
|
|
9
|
-
console.log(
|
|
10
|
-
await nut_js_1.mouse.move([
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
console.log(
|
|
14
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
15
|
-
console.log("Clicking now");
|
|
9
|
+
console.log('Moving mouse to:', centerX, centerY);
|
|
10
|
+
await nut_js_1.mouse.move([new nut_js_1.Point(centerX, centerY)]);
|
|
11
|
+
console.log('Mouse moved');
|
|
12
|
+
await new Promise((r) => setTimeout(r, 2000));
|
|
13
|
+
console.log('Clicking now');
|
|
16
14
|
await nut_js_1.mouse.click(nut_js_1.Button.LEFT);
|
|
17
|
-
console.log(
|
|
15
|
+
console.log('Click done');
|
|
18
16
|
}
|
package/dist/src/screen.js
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
|
|
2
|
-
var __importDefault =
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
'use strict';
|
|
2
|
+
var __importDefault =
|
|
3
|
+
(this && this.__importDefault) ||
|
|
4
|
+
function (mod) {
|
|
5
|
+
return mod && mod.__esModule ? mod : { default: mod };
|
|
6
|
+
};
|
|
7
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
6
8
|
exports.captureScreen = captureScreen;
|
|
7
|
-
const screenshot_desktop_1 = __importDefault(require(
|
|
8
|
-
const pngjs_1 = require(
|
|
9
|
-
const cv = require(
|
|
9
|
+
const screenshot_desktop_1 = __importDefault(require('screenshot-desktop'));
|
|
10
|
+
const pngjs_1 = require('pngjs');
|
|
11
|
+
const cv = require('opencv.js');
|
|
10
12
|
async function captureScreen() {
|
|
11
13
|
const buffer = await (0, screenshot_desktop_1.default)({
|
|
12
|
-
format:
|
|
14
|
+
format: 'png',
|
|
13
15
|
});
|
|
14
16
|
const png = pngjs_1.PNG.sync.read(buffer);
|
|
15
17
|
return cv.matFromImageData({
|
|
16
18
|
data: png.data,
|
|
17
19
|
width: png.width,
|
|
18
|
-
height: png.height
|
|
20
|
+
height: png.height,
|
|
19
21
|
});
|
|
20
22
|
}
|
package/dist/src/types.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports,
|
|
1
|
+
'use strict';
|
|
2
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function loadTemplateMetadata(path: string): any;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.loadTemplateMetadata = loadTemplateMetadata;
|
|
7
|
+
const fs_1 = __importDefault(require("fs"));
|
|
8
|
+
const metadataCache = new Map();
|
|
9
|
+
function loadTemplateMetadata(path) {
|
|
10
|
+
const metadataPath = path.replace(/\.png$/i, '.properties.json');
|
|
11
|
+
if (metadataCache.has(metadataPath)) {
|
|
12
|
+
return metadataCache.get(metadataPath);
|
|
13
|
+
}
|
|
14
|
+
if (!fs_1.default.existsSync(metadataPath)) {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
const metadata = JSON.parse(fs_1.default.readFileSync(metadataPath, 'utf-8'));
|
|
18
|
+
metadataCache.set(metadataPath, metadata);
|
|
19
|
+
return metadata;
|
|
20
|
+
}
|
package/dist/text.d.ts
CHANGED
package/dist/text.js
CHANGED
|
@@ -10,11 +10,11 @@ async function findText(text, index = 0, region) {
|
|
|
10
10
|
if (!tsv) {
|
|
11
11
|
return null;
|
|
12
12
|
}
|
|
13
|
-
const rows = tsv.split(
|
|
13
|
+
const rows = tsv.split('\n');
|
|
14
14
|
const lineGroups = {};
|
|
15
15
|
for (const row of rows) {
|
|
16
|
-
const cols = row.split(
|
|
17
|
-
if (cols[0] !==
|
|
16
|
+
const cols = row.split('\t');
|
|
17
|
+
if (cols[0] !== '5') {
|
|
18
18
|
continue;
|
|
19
19
|
}
|
|
20
20
|
const wordText = cols[11];
|
|
@@ -49,14 +49,14 @@ async function findText(text, index = 0, region) {
|
|
|
49
49
|
//const combinedText = words.map(w => w.text).join(" ");
|
|
50
50
|
const combinedText = words
|
|
51
51
|
.map((w) => w.text)
|
|
52
|
-
.join(
|
|
53
|
-
.replace(/\s+/g,
|
|
52
|
+
.join(' ')
|
|
53
|
+
.replace(/\s+/g, ' ')
|
|
54
54
|
.trim();
|
|
55
55
|
if (combinedText
|
|
56
56
|
.toLowerCase()
|
|
57
|
-
.replace(/[^a-z0-9 ]/gi,
|
|
58
|
-
.includes(text.toLowerCase().replace(/[^a-z0-9 ]/gi,
|
|
59
|
-
const matchedWords = words.filter((w) => searchWords.includes(w.text.toLowerCase().replace(/[^a-z0-9]/gi,
|
|
57
|
+
.replace(/[^a-z0-9 ]/gi, '')
|
|
58
|
+
.includes(text.toLowerCase().replace(/[^a-z0-9 ]/gi, ''))) {
|
|
59
|
+
const matchedWords = words.filter((w) => searchWords.includes(w.text.toLowerCase().replace(/[^a-z0-9]/gi, '')));
|
|
60
60
|
const regionWords = matchedWords.length > 0 ? matchedWords : words;
|
|
61
61
|
const minX = Math.min(...regionWords.map((w) => w.x));
|
|
62
62
|
const minY = Math.min(...regionWords.map((w) => w.y));
|