askui 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/core/cache/cache-config.d.ts +5 -0
- package/dist/cjs/core/cache/cache-config.js +2 -0
- package/dist/cjs/core/cache/cache-entry-reference.d.ts +7 -0
- package/dist/cjs/core/cache/cache-entry-reference.js +20 -0
- package/dist/cjs/core/cache/cache-entry.d.ts +11 -0
- package/dist/cjs/core/cache/cache-entry.js +47 -0
- package/dist/cjs/core/cache/cache-interface.d.ts +11 -0
- package/dist/cjs/core/cache/cache-interface.js +2 -0
- package/dist/cjs/core/cache/cache-manager.d.ts +24 -0
- package/dist/cjs/core/cache/cache-manager.js +145 -0
- package/dist/cjs/core/cache/cahe-file.d.ts +19 -0
- package/dist/cjs/core/cache/cahe-file.js +128 -0
- package/dist/cjs/core/cache/dummy-cache-manager.d.ts +12 -0
- package/dist/cjs/core/cache/dummy-cache-manager.js +27 -0
- package/dist/cjs/core/cache/image-reference.d.ts +10 -0
- package/dist/cjs/core/cache/image-reference.js +40 -0
- package/dist/cjs/core/cache/index.d.ts +7 -0
- package/dist/cjs/core/cache/index.js +13 -0
- package/dist/cjs/core/model/annotation-result/boundary-box.js +1 -1
- package/dist/cjs/core/model/custom-element.d.ts +1 -0
- package/dist/cjs/core/model/custom-element.js +3 -0
- package/dist/cjs/core/ui-control-commands/action.d.ts +1 -0
- package/dist/cjs/core/ui-control-commands/action.js +10 -2
- package/dist/cjs/core/ui-control-commands/control-command.d.ts +1 -0
- package/dist/cjs/core/ui-control-commands/control-command.js +7 -0
- package/dist/cjs/execution/dsl.d.ts +12 -5
- package/dist/cjs/execution/dsl.js +30 -15
- package/dist/cjs/execution/execution-runtime.d.ts +1 -1
- package/dist/cjs/execution/execution-runtime.js +21 -17
- package/dist/cjs/execution/inference-client.d.ts +5 -3
- package/dist/cjs/execution/inference-client.js +22 -3
- package/dist/cjs/execution/ui-control-client-dependency-builder.js +6 -1
- package/dist/cjs/execution/ui-control-client.d.ts +2 -1
- package/dist/cjs/execution/ui-control-client.js +28 -11
- package/dist/cjs/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/cjs/utils/base_64_image/base-64-image.d.ts +2 -0
- package/dist/cjs/utils/base_64_image/base-64-image.js +27 -4
- package/dist/esm/core/cache/cache-config.d.ts +5 -0
- package/dist/esm/core/cache/cache-config.js +1 -0
- package/dist/esm/core/cache/cache-entry-reference.d.ts +7 -0
- package/dist/esm/core/cache/cache-entry-reference.js +16 -0
- package/dist/esm/core/cache/cache-entry.d.ts +11 -0
- package/dist/esm/core/cache/cache-entry.js +43 -0
- package/dist/esm/core/cache/cache-interface.d.ts +11 -0
- package/dist/esm/core/cache/cache-interface.js +1 -0
- package/dist/esm/core/cache/cache-manager.d.ts +24 -0
- package/dist/esm/core/cache/cache-manager.js +141 -0
- package/dist/esm/core/cache/cahe-file.d.ts +19 -0
- package/dist/esm/core/cache/cahe-file.js +121 -0
- package/dist/esm/core/cache/dummy-cache-manager.d.ts +12 -0
- package/dist/esm/core/cache/dummy-cache-manager.js +23 -0
- package/dist/esm/core/cache/image-reference.d.ts +10 -0
- package/dist/esm/core/cache/image-reference.js +36 -0
- package/dist/esm/core/cache/index.d.ts +7 -0
- package/dist/esm/core/cache/index.js +5 -0
- package/dist/esm/core/model/annotation-result/boundary-box.js +1 -1
- package/dist/esm/core/model/custom-element.d.ts +1 -0
- package/dist/esm/core/model/custom-element.js +3 -0
- package/dist/esm/core/ui-control-commands/action.d.ts +1 -0
- package/dist/esm/core/ui-control-commands/action.js +10 -2
- package/dist/esm/core/ui-control-commands/control-command.d.ts +1 -0
- package/dist/esm/core/ui-control-commands/control-command.js +7 -0
- package/dist/esm/execution/dsl.d.ts +12 -5
- package/dist/esm/execution/dsl.js +30 -15
- package/dist/esm/execution/execution-runtime.d.ts +1 -1
- package/dist/esm/execution/execution-runtime.js +21 -17
- package/dist/esm/execution/inference-client.d.ts +5 -3
- package/dist/esm/execution/inference-client.js +22 -3
- package/dist/esm/execution/ui-control-client-dependency-builder.js +6 -1
- package/dist/esm/execution/ui-control-client.d.ts +2 -1
- package/dist/esm/execution/ui-control-client.js +28 -11
- package/dist/esm/execution/ui-controller-client-interface.d.ts +2 -0
- package/dist/esm/utils/base_64_image/base-64-image.d.ts +2 -0
- package/dist/esm/utils/base_64_image/base-64-image.js +27 -4
- package/package.json +1 -1
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { ControlCommand } from '../ui-control-commands';
|
|
2
|
+
import { CacheEntryReference } from './cache-entry-reference';
|
|
3
|
+
import { logger } from '../../lib';
|
|
4
|
+
export class CacheEntry {
|
|
5
|
+
constructor(alwaysValid, controlCommand, reference, createdAt = new Date()) {
|
|
6
|
+
this.alwaysValid = alwaysValid;
|
|
7
|
+
this.controlCommand = controlCommand;
|
|
8
|
+
this.reference = reference;
|
|
9
|
+
this.createdAt = createdAt;
|
|
10
|
+
}
|
|
11
|
+
static fromJson(json) {
|
|
12
|
+
try {
|
|
13
|
+
if (json === undefined) {
|
|
14
|
+
throw new Error('Cache entry is undefined');
|
|
15
|
+
}
|
|
16
|
+
if (json.controlCommand === undefined) {
|
|
17
|
+
throw new Error('the key "controlCommand" is required');
|
|
18
|
+
}
|
|
19
|
+
if (json.createdAt === undefined) {
|
|
20
|
+
throw new Error('the key "createdAt" is required');
|
|
21
|
+
}
|
|
22
|
+
if (json.alwaysValid === undefined) {
|
|
23
|
+
throw new Error('the key "alwaysValid" is required');
|
|
24
|
+
}
|
|
25
|
+
return new CacheEntry(json.alwaysValid, ControlCommand.fromJson(json.controlCommand), json.reference ? CacheEntryReference.fromJson(json.reference) : undefined, new Date(json.createdAt));
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
logger.error(`Error deserializing cache entry: ${error}`);
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
toJson() {
|
|
33
|
+
const jsonObject = {
|
|
34
|
+
alwaysValid: this.alwaysValid,
|
|
35
|
+
controlCommand: this.controlCommand.toJson(),
|
|
36
|
+
createdAt: this.createdAt,
|
|
37
|
+
};
|
|
38
|
+
if (this.reference !== undefined) {
|
|
39
|
+
jsonObject['reference'] = this.reference.toJson();
|
|
40
|
+
}
|
|
41
|
+
return jsonObject;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { CustomElement } from '../model/custom-element';
|
|
2
|
+
import { ControlCommand } from '../ui-control-commands';
|
|
3
|
+
import { CacheEntry } from './cache-entry';
|
|
4
|
+
export interface CacheInterface {
|
|
5
|
+
loadFromFile(): void;
|
|
6
|
+
saveToFile(): void;
|
|
7
|
+
add(key: string, value: CacheEntry): void;
|
|
8
|
+
addCacheEntryFromControlCommand(instruction: string, controlCommand: ControlCommand, customElements: CustomElement[], image?: string): Promise<void>;
|
|
9
|
+
isImageRequired(instruction: string, customElements: CustomElement[]): boolean | undefined;
|
|
10
|
+
getCachedControlCommand(instruction: string, customElements: CustomElement[], image?: string): Promise<ControlCommand | undefined>;
|
|
11
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { ControlCommand } from '../ui-control-commands';
|
|
2
|
+
import { CacheInterface } from './cache-interface';
|
|
3
|
+
import { CacheEntry } from './cache-entry';
|
|
4
|
+
import { CacheConfig, ValidationType } from './cache-config';
|
|
5
|
+
import { CustomElement } from '../model/custom-element';
|
|
6
|
+
export declare class CacheManager implements CacheInterface {
|
|
7
|
+
private relevantData;
|
|
8
|
+
validationType: ValidationType;
|
|
9
|
+
private cacheFile;
|
|
10
|
+
private readonly referenceWidth;
|
|
11
|
+
private readonly referenceHeight;
|
|
12
|
+
private readonly defaultValidationType;
|
|
13
|
+
constructor(config: CacheConfig);
|
|
14
|
+
loadFromFile(): void;
|
|
15
|
+
saveToFile(): void;
|
|
16
|
+
add(key: string, value: CacheEntry): void;
|
|
17
|
+
addCacheEntryFromControlCommand(instruction: string, controlCommand: ControlCommand, customElements: CustomElement[], image?: string): Promise<void>;
|
|
18
|
+
isImageRequired(instruction: string, customElements: CustomElement[]): boolean | undefined;
|
|
19
|
+
getCachedControlCommand(instruction: string, customElements: CustomElement[], image?: string): Promise<ControlCommand | undefined>;
|
|
20
|
+
private encodeKey;
|
|
21
|
+
private getCacheEntriesByKey;
|
|
22
|
+
private getValidCacheEntry;
|
|
23
|
+
private validateAccordingToPixelPerfect;
|
|
24
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { ControlCommandCode } from '../ui-control-commands';
|
|
11
|
+
import { InputEvent } from '../ui-control-commands/input-event';
|
|
12
|
+
import { Base64Image } from '../../utils/base_64_image/base-64-image';
|
|
13
|
+
import { CacheEntry } from './cache-entry';
|
|
14
|
+
import { ImageReference } from './image-reference';
|
|
15
|
+
import { CacheEntryReference } from './cache-entry-reference';
|
|
16
|
+
import { CacheFile } from './cahe-file';
|
|
17
|
+
export class CacheManager {
|
|
18
|
+
constructor(config) {
|
|
19
|
+
var _a;
|
|
20
|
+
this.relevantData = {};
|
|
21
|
+
this.referenceWidth = 32;
|
|
22
|
+
this.referenceHeight = 16;
|
|
23
|
+
this.defaultValidationType = 'PixelPerfect';
|
|
24
|
+
this.validationType = (_a = config.validationType) !== null && _a !== void 0 ? _a : this.defaultValidationType;
|
|
25
|
+
this.cacheFile = new CacheFile(config.cacheFilePath);
|
|
26
|
+
}
|
|
27
|
+
loadFromFile() {
|
|
28
|
+
this.cacheFile.loadFromFile();
|
|
29
|
+
this.relevantData = this.cacheFile.getDataForValidationType(this.validationType);
|
|
30
|
+
}
|
|
31
|
+
saveToFile() {
|
|
32
|
+
this.cacheFile.saveToFile(this.validationType, this.relevantData);
|
|
33
|
+
}
|
|
34
|
+
add(key, value) {
|
|
35
|
+
if (this.relevantData[key] === undefined) {
|
|
36
|
+
this.relevantData[key] = [];
|
|
37
|
+
}
|
|
38
|
+
this.relevantData[key].push(value);
|
|
39
|
+
}
|
|
40
|
+
addCacheEntryFromControlCommand(instruction, controlCommand, customElements, image) {
|
|
41
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
42
|
+
let imageReference;
|
|
43
|
+
if (image !== undefined) {
|
|
44
|
+
const moveAction = controlCommand.actions.find((action) => action.inputEvent === InputEvent.MOUSE_MOVE);
|
|
45
|
+
const x = moveAction === null || moveAction === void 0 ? void 0 : moveAction.position.x;
|
|
46
|
+
const y = moveAction === null || moveAction === void 0 ? void 0 : moveAction.position.y;
|
|
47
|
+
if (x !== undefined && y !== undefined) {
|
|
48
|
+
const xTopLeft = Math.max(0, (x - this.referenceWidth / 2));
|
|
49
|
+
const yTopLeft = Math.max(0, (y - this.referenceHeight / 2));
|
|
50
|
+
const base64Image = yield Base64Image.fromString(image);
|
|
51
|
+
const croppedImage = yield base64Image.cropRegion(xTopLeft, yTopLeft, this.referenceWidth, this.referenceHeight);
|
|
52
|
+
imageReference = new ImageReference(croppedImage.toString(true), this.referenceWidth, this.referenceHeight, xTopLeft, yTopLeft);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const cacheEntry = new CacheEntry(image === undefined, controlCommand, new CacheEntryReference(imageReference));
|
|
56
|
+
const cacheKey = this.encodeKey(instruction, customElements);
|
|
57
|
+
this.add(cacheKey, cacheEntry);
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
isImageRequired(instruction, customElements) {
|
|
61
|
+
const cacheKey = this.encodeKey(instruction, customElements);
|
|
62
|
+
const cacheEntries = this.getCacheEntriesByKey(cacheKey);
|
|
63
|
+
if (cacheEntries.length === 0) {
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
return cacheEntries.find((entry) => entry.alwaysValid === false) !== undefined;
|
|
67
|
+
}
|
|
68
|
+
getCachedControlCommand(instruction, customElements, image) {
|
|
69
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
70
|
+
const cacheKey = this.encodeKey(instruction, customElements);
|
|
71
|
+
const cacheEntry = yield this.getValidCacheEntry(cacheKey, image);
|
|
72
|
+
if (cacheEntry === undefined || cacheEntry.controlCommand.code !== ControlCommandCode.OK) {
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
return cacheEntry.controlCommand;
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
// eslint-disable-next-line class-methods-use-this
|
|
79
|
+
encodeKey(instruction, customElements) {
|
|
80
|
+
let key = instruction;
|
|
81
|
+
if (customElements.length > 0) {
|
|
82
|
+
key += ` with CustomElements:${customElements.map((element, index) => `${index}:${element.asString()}`).join(',')}`;
|
|
83
|
+
}
|
|
84
|
+
return key;
|
|
85
|
+
}
|
|
86
|
+
getCacheEntriesByKey(key) {
|
|
87
|
+
if (this.relevantData[key] === undefined) {
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
const entries = this.relevantData[key];
|
|
91
|
+
if (entries === undefined) {
|
|
92
|
+
return [];
|
|
93
|
+
}
|
|
94
|
+
return entries;
|
|
95
|
+
}
|
|
96
|
+
getValidCacheEntry(key, screenshot) {
|
|
97
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
98
|
+
const cacheEntries = this.getCacheEntriesByKey(key);
|
|
99
|
+
if (cacheEntries.length === 0) {
|
|
100
|
+
return undefined;
|
|
101
|
+
}
|
|
102
|
+
if (this.validationType === 'PixelPerfect') {
|
|
103
|
+
/* eslint-disable no-restricted-syntax, no-await-in-loop */
|
|
104
|
+
for (const cacheEntry of cacheEntries) {
|
|
105
|
+
if (cacheEntry.alwaysValid) {
|
|
106
|
+
return cacheEntry;
|
|
107
|
+
}
|
|
108
|
+
const isValid = yield this.validateAccordingToPixelPerfect(cacheEntry, screenshot);
|
|
109
|
+
if (isValid) {
|
|
110
|
+
/* eslint-enable no-restricted-syntax, no-await-in-loop */
|
|
111
|
+
return cacheEntry;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/* eslint-enable no-restricted-syntax, no-await-in-loop */
|
|
115
|
+
}
|
|
116
|
+
return undefined;
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
// eslint-disable-next-line class-methods-use-this
|
|
120
|
+
validateAccordingToPixelPerfect(cacheEntry, screenshot) {
|
|
121
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
122
|
+
var _a;
|
|
123
|
+
if (screenshot === undefined) {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
if (((_a = cacheEntry.reference) === null || _a === void 0 ? void 0 : _a.image) === undefined) {
|
|
127
|
+
return false;
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
const referenceImage = yield Base64Image.fromString(screenshot);
|
|
131
|
+
const croppedScreenshot = yield referenceImage.cropRegion(cacheEntry.reference.image.xTopLeft, cacheEntry.reference.image.yTopLeft, cacheEntry.reference.image.width, cacheEntry.reference.image.height);
|
|
132
|
+
const croppedReferenceImageString = croppedScreenshot.toString(true);
|
|
133
|
+
const isValid = croppedReferenceImageString === cacheEntry.reference.image.base64Image;
|
|
134
|
+
return isValid;
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { CacheEntry } from './cache-entry';
|
|
2
|
+
import { ValidationType } from './cache-config';
|
|
3
|
+
export type CacheDataByInstruction = Record<string, CacheEntry[]>;
|
|
4
|
+
export type CacheDataByValidationType = Record<string, CacheDataByInstruction>;
|
|
5
|
+
export declare class CacheFile {
|
|
6
|
+
private createAt;
|
|
7
|
+
private version;
|
|
8
|
+
private type;
|
|
9
|
+
private data;
|
|
10
|
+
private filePath;
|
|
11
|
+
constructor(filePath?: string);
|
|
12
|
+
loadFromFileVersion1(json: CacheFile): void;
|
|
13
|
+
loadFromFile(): void;
|
|
14
|
+
getDataForValidationType(validationType: ValidationType): CacheDataByInstruction;
|
|
15
|
+
getData(): CacheDataByValidationType;
|
|
16
|
+
saveToFile(validationType: ValidationType, relevantData: CacheDataByInstruction): void;
|
|
17
|
+
private setRelevantData;
|
|
18
|
+
private asJsonObject;
|
|
19
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { logger } from '../../lib';
|
|
4
|
+
import { CacheEntry } from './cache-entry';
|
|
5
|
+
export class CacheFile {
|
|
6
|
+
constructor(filePath) {
|
|
7
|
+
this.createAt = new Date();
|
|
8
|
+
this.version = 1;
|
|
9
|
+
this.type = 'AskUI-Cache';
|
|
10
|
+
this.data = {};
|
|
11
|
+
this.filePath = filePath;
|
|
12
|
+
}
|
|
13
|
+
loadFromFileVersion1(json) {
|
|
14
|
+
if (json.version !== 1) {
|
|
15
|
+
throw new Error(`Unsupported key 'version' in the cache file: '${json.version}'. Only version 1 is supported.`);
|
|
16
|
+
}
|
|
17
|
+
if (json.data === undefined) {
|
|
18
|
+
throw new Error("Key 'data' is required in the cache file.");
|
|
19
|
+
}
|
|
20
|
+
if (json.createAt === undefined) {
|
|
21
|
+
throw new Error("Key 'createAt' is required in the cache file.");
|
|
22
|
+
}
|
|
23
|
+
if (json.version === undefined) {
|
|
24
|
+
throw new Error("Key 'version' is required in the cache file.");
|
|
25
|
+
}
|
|
26
|
+
if (json.type !== 'AskUI-Cache') {
|
|
27
|
+
throw new Error(`Unsupported key 'type' in the cache file: '${json.type}'. Only 'AskUI-Cache' is supported.`);
|
|
28
|
+
}
|
|
29
|
+
this.createAt = new Date(json.createAt);
|
|
30
|
+
this.version = json.version;
|
|
31
|
+
this.type = json.type;
|
|
32
|
+
this.data = Object.keys(json.data).reduce((acc, validationType) => {
|
|
33
|
+
var _a, _b;
|
|
34
|
+
const nextAcc = Object.assign({}, acc);
|
|
35
|
+
const validationEntries = (_b = (_a = json.data) === null || _a === void 0 ? void 0 : _a[validationType]) !== null && _b !== void 0 ? _b : {};
|
|
36
|
+
nextAcc[validationType] = Object.keys(validationEntries).reduce((innerAcc, instruction) => {
|
|
37
|
+
var _a;
|
|
38
|
+
const nextInnerAcc = Object.assign({}, innerAcc);
|
|
39
|
+
const rawEntries = (_a = validationEntries[instruction]) !== null && _a !== void 0 ? _a : [];
|
|
40
|
+
nextInnerAcc[instruction] = rawEntries
|
|
41
|
+
.map((entry) => CacheEntry.fromJson(entry))
|
|
42
|
+
.filter((entry) => entry !== undefined);
|
|
43
|
+
return nextInnerAcc;
|
|
44
|
+
}, {});
|
|
45
|
+
return nextAcc;
|
|
46
|
+
}, {});
|
|
47
|
+
}
|
|
48
|
+
loadFromFile() {
|
|
49
|
+
if (this.filePath === undefined) {
|
|
50
|
+
logger.debug('The cache file path is not set. Skipping the cache load.');
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
logger.debug(`Loading the cache from file '${this.filePath}'.`);
|
|
54
|
+
if (!fs.existsSync(this.filePath)) {
|
|
55
|
+
logger.warn(`The cache file does not exist: '${this.filePath}'. Skipping the cache load.`);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
const data = fs.readFileSync(this.filePath, 'utf8');
|
|
59
|
+
const json = JSON.parse(data);
|
|
60
|
+
if (json.version !== 1) {
|
|
61
|
+
throw new Error(`Unsupported key 'version' in the cache file: '${json.version}'. Only version 1 is supported.`);
|
|
62
|
+
}
|
|
63
|
+
this.loadFromFileVersion1(json);
|
|
64
|
+
}
|
|
65
|
+
getDataForValidationType(validationType) {
|
|
66
|
+
var _a;
|
|
67
|
+
return (_a = this.data[validationType]) !== null && _a !== void 0 ? _a : {};
|
|
68
|
+
}
|
|
69
|
+
getData() {
|
|
70
|
+
return this.data;
|
|
71
|
+
}
|
|
72
|
+
saveToFile(validationType, relevantData) {
|
|
73
|
+
if (this.filePath === undefined) {
|
|
74
|
+
logger.debug('The cache file path is not set. Skipping the cache save.');
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
try {
|
|
78
|
+
logger.debug(`Saving the cache to file '${this.filePath}'.`);
|
|
79
|
+
this.setRelevantData(validationType, relevantData);
|
|
80
|
+
const jsonObject = this.asJsonObject();
|
|
81
|
+
const jsonString = JSON.stringify(jsonObject, null, 2);
|
|
82
|
+
// create the directory if it doesn't exist
|
|
83
|
+
if (!fs.existsSync(path.dirname(this.filePath))) {
|
|
84
|
+
fs.mkdirSync(path.dirname(this.filePath), { recursive: true });
|
|
85
|
+
}
|
|
86
|
+
fs.writeFileSync(this.filePath, jsonString, 'utf8');
|
|
87
|
+
logger.info(`The cache was saved successfully to '${this.filePath}'.`);
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
logger.error(`An error occurred while saving the cache to file '${this.filePath}': '${error}'`);
|
|
91
|
+
throw error;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
setRelevantData(validationType, relevantData) {
|
|
95
|
+
this.data[validationType] = relevantData;
|
|
96
|
+
}
|
|
97
|
+
asJsonObject() {
|
|
98
|
+
return {
|
|
99
|
+
createAt: this.createAt.toISOString(),
|
|
100
|
+
data: Object.keys(this.data).reduce((acc, validationType) => {
|
|
101
|
+
const validationData = this.data[validationType];
|
|
102
|
+
if (validationData === undefined) {
|
|
103
|
+
return acc;
|
|
104
|
+
}
|
|
105
|
+
const nextAcc = Object.assign({}, acc);
|
|
106
|
+
nextAcc[validationType] = Object.keys(validationData).reduce((innerAcc, instruction) => {
|
|
107
|
+
const entries = validationData[instruction];
|
|
108
|
+
if (entries !== undefined) {
|
|
109
|
+
const nextInnerAcc = Object.assign({}, innerAcc);
|
|
110
|
+
nextInnerAcc[instruction] = entries;
|
|
111
|
+
return nextInnerAcc;
|
|
112
|
+
}
|
|
113
|
+
return innerAcc;
|
|
114
|
+
}, {});
|
|
115
|
+
return nextAcc;
|
|
116
|
+
}, {}),
|
|
117
|
+
type: this.type,
|
|
118
|
+
version: this.version,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { CacheInterface } from './cache-interface';
|
|
2
|
+
import { CacheEntry } from './cache-entry';
|
|
3
|
+
import { ControlCommand } from '../ui-control-commands';
|
|
4
|
+
import { CustomElement } from '../model/custom-element';
|
|
5
|
+
export declare class DummyCacheManager implements CacheInterface {
|
|
6
|
+
add(_key: string, _value: CacheEntry): void;
|
|
7
|
+
addCacheEntryFromControlCommand(_instruction: string, _controlCommand: ControlCommand, _customElements: CustomElement[], _image?: string): Promise<void>;
|
|
8
|
+
isImageRequired(_instruction: string, _customElements: CustomElement[]): boolean | undefined;
|
|
9
|
+
getCachedControlCommand(_instruction: string, _customElements: CustomElement[], _image?: string): Promise<ControlCommand | undefined>;
|
|
10
|
+
loadFromFile(): void;
|
|
11
|
+
saveToFile(): void;
|
|
12
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export class DummyCacheManager {
|
|
2
|
+
// eslint-disable-next-line class-methods-use-this
|
|
3
|
+
add(_key, _value) {
|
|
4
|
+
}
|
|
5
|
+
// eslint-disable-next-line class-methods-use-this
|
|
6
|
+
addCacheEntryFromControlCommand(_instruction, _controlCommand, _customElements, _image) {
|
|
7
|
+
return Promise.resolve();
|
|
8
|
+
}
|
|
9
|
+
// eslint-disable-next-line class-methods-use-this
|
|
10
|
+
isImageRequired(_instruction, _customElements) {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
// eslint-disable-next-line class-methods-use-this
|
|
14
|
+
getCachedControlCommand(_instruction, _customElements, _image) {
|
|
15
|
+
return Promise.resolve(undefined);
|
|
16
|
+
}
|
|
17
|
+
// eslint-disable-next-line class-methods-use-this
|
|
18
|
+
loadFromFile() {
|
|
19
|
+
}
|
|
20
|
+
// eslint-disable-next-line class-methods-use-this
|
|
21
|
+
saveToFile() {
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare class ImageReference {
|
|
2
|
+
base64Image: string;
|
|
3
|
+
width: number;
|
|
4
|
+
height: number;
|
|
5
|
+
xTopLeft: number;
|
|
6
|
+
yTopLeft: number;
|
|
7
|
+
constructor(base64Image: string, width: number, height: number, xTopLeft: number, yTopLeft: number);
|
|
8
|
+
static fromJson(json: ImageReference): ImageReference;
|
|
9
|
+
toJson(): Record<string, unknown>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export class ImageReference {
|
|
2
|
+
constructor(base64Image, width, height, xTopLeft, yTopLeft) {
|
|
3
|
+
this.base64Image = base64Image;
|
|
4
|
+
this.width = width;
|
|
5
|
+
this.height = height;
|
|
6
|
+
this.xTopLeft = xTopLeft;
|
|
7
|
+
this.yTopLeft = yTopLeft;
|
|
8
|
+
}
|
|
9
|
+
static fromJson(json) {
|
|
10
|
+
if (json.base64Image === undefined) {
|
|
11
|
+
throw new Error('the key "base64Image" is required');
|
|
12
|
+
}
|
|
13
|
+
if (json.width === undefined) {
|
|
14
|
+
throw new Error('the key "width" is required');
|
|
15
|
+
}
|
|
16
|
+
if (json.height === undefined) {
|
|
17
|
+
throw new Error('the key "height" is required');
|
|
18
|
+
}
|
|
19
|
+
if (json.xTopLeft === undefined) {
|
|
20
|
+
throw new Error('the key "xTopLeft" is required');
|
|
21
|
+
}
|
|
22
|
+
if (json.yTopLeft === undefined) {
|
|
23
|
+
throw new Error('the key "yTopLeft" is required');
|
|
24
|
+
}
|
|
25
|
+
return new ImageReference(json.base64Image, json.width, json.height, json.xTopLeft, json.yTopLeft);
|
|
26
|
+
}
|
|
27
|
+
toJson() {
|
|
28
|
+
return {
|
|
29
|
+
base64Image: this.base64Image,
|
|
30
|
+
height: this.height,
|
|
31
|
+
width: this.width,
|
|
32
|
+
xTopLeft: this.xTopLeft,
|
|
33
|
+
yTopLeft: this.yTopLeft,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { ImageReference } from './image-reference';
|
|
2
|
+
export { CacheEntryReference } from './cache-entry-reference';
|
|
3
|
+
export { CacheEntry } from './cache-entry';
|
|
4
|
+
export { CacheConfig, ValidationType } from './cache-config';
|
|
5
|
+
export { CacheInterface } from './cache-interface';
|
|
6
|
+
export { DummyCacheManager } from './dummy-cache-manager';
|
|
7
|
+
export { CacheManager } from './cache-manager';
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { ImageReference } from './image-reference';
|
|
2
|
+
export { CacheEntryReference } from './cache-entry-reference';
|
|
3
|
+
export { CacheEntry } from './cache-entry';
|
|
4
|
+
export { DummyCacheManager } from './dummy-cache-manager';
|
|
5
|
+
export { CacheManager } from './cache-manager';
|
|
@@ -17,7 +17,7 @@ export class BoundingBox {
|
|
|
17
17
|
this.ymax = ymax;
|
|
18
18
|
}
|
|
19
19
|
static fromJson(boundinBox, resizeRatio = 1) {
|
|
20
|
-
return new BoundingBox(boundinBox.xmin * resizeRatio, boundinBox.ymin * resizeRatio, boundinBox.xmax * resizeRatio, boundinBox.ymax * resizeRatio);
|
|
20
|
+
return new BoundingBox(Math.round(boundinBox.xmin * resizeRatio), Math.round(boundinBox.ymin * resizeRatio), Math.round(boundinBox.xmax * resizeRatio), Math.round(boundinBox.ymax * resizeRatio));
|
|
21
21
|
}
|
|
22
22
|
/**
|
|
23
23
|
*
|
|
@@ -43,6 +43,9 @@ export class CustomElement {
|
|
|
43
43
|
throw new ValidationError(e.errors.join(', '));
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
|
+
asString() {
|
|
47
|
+
return `name:${this.name}-customImage:${this.customImage}-threshold:${this.threshold}-stopThreshold:${this.stopThreshold}-rotationDegreePerStep:${this.rotationDegreePerStep}-imageCompareFormat:${this.imageCompareFormat}-mask:${this.mask}`;
|
|
48
|
+
}
|
|
46
49
|
}
|
|
47
50
|
CustomElement.schema = object({
|
|
48
51
|
mask: array().optional().min(3, 'mask must contain at least 3 points'),
|
|
@@ -8,8 +8,16 @@ export class Action {
|
|
|
8
8
|
}
|
|
9
9
|
static fromJson(action, resizeRatio = 1) {
|
|
10
10
|
return new Action(InputEvent[action.inputEvent], {
|
|
11
|
-
x: action.position.x * resizeRatio,
|
|
12
|
-
y: action.position.y * resizeRatio,
|
|
11
|
+
x: Math.round(action.position.x * resizeRatio),
|
|
12
|
+
y: Math.round(action.position.y * resizeRatio),
|
|
13
13
|
}, action.text, action.parameters ? action.parameters : {});
|
|
14
14
|
}
|
|
15
|
+
toJson() {
|
|
16
|
+
return {
|
|
17
|
+
inputEvent: this.inputEvent,
|
|
18
|
+
parameters: this.parameters,
|
|
19
|
+
position: this.position,
|
|
20
|
+
text: this.text,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
15
23
|
}
|
|
@@ -15,4 +15,11 @@ export class ControlCommand {
|
|
|
15
15
|
this.actions = this.actions.map((action) => ([InputEvent.TYPE, InputEvent.TYPE_TEXT].includes(action.inputEvent)
|
|
16
16
|
? new Action(action.inputEvent, action.position, text, action.parameters) : action));
|
|
17
17
|
}
|
|
18
|
+
toJson() {
|
|
19
|
+
return {
|
|
20
|
+
actions: this.actions.map((action) => action.toJson()),
|
|
21
|
+
code: this.code,
|
|
22
|
+
tryToRepeat: this.tryToRepeat,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
18
25
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { CustomElementJson } from '../core/model/custom-element-json';
|
|
2
2
|
import { DetectedElement } from '../core/model/annotation-result/detected-element';
|
|
3
3
|
import { ModelCompositionBranch } from './model-composition-branch';
|
|
4
|
+
import { RetryStrategy } from './retry-strategies/retry-strategy';
|
|
4
5
|
export declare enum Separators {
|
|
5
6
|
STRING = "<|string|>"
|
|
6
7
|
}
|
|
@@ -17,22 +18,27 @@ export interface CommandExecutorContext {
|
|
|
17
18
|
customElementsJson: CustomElementJson[];
|
|
18
19
|
aiElementNames: string[];
|
|
19
20
|
}
|
|
21
|
+
export interface ExecOptions {
|
|
22
|
+
modelComposition?: ModelCompositionBranch[];
|
|
23
|
+
skipCache?: boolean;
|
|
24
|
+
retryStrategy?: RetryStrategy;
|
|
25
|
+
}
|
|
20
26
|
declare abstract class FluentBase {
|
|
21
27
|
protected prev?: FluentBase | undefined;
|
|
22
28
|
constructor(prev?: FluentBase | undefined);
|
|
23
29
|
protected _textStr: string;
|
|
24
30
|
protected _params: Map<string, unknown>;
|
|
25
31
|
protected static addParams(paramsList: Map<string, unknown[]>, params: Map<string, unknown>): Map<string, unknown[]>;
|
|
26
|
-
protected fluentCommandStringBuilder(modelComposition
|
|
32
|
+
protected fluentCommandStringBuilder(modelComposition?: ModelCompositionBranch[], skipCache?: boolean, retryStrategy?: RetryStrategy, currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<void>;
|
|
27
33
|
protected getterStringBuilder(currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<DetectedElement[]>;
|
|
28
34
|
protected get textStr(): string;
|
|
29
35
|
protected get params(): Map<string, unknown>;
|
|
30
36
|
}
|
|
31
37
|
export interface Executable {
|
|
32
|
-
exec(): Promise<void>;
|
|
38
|
+
exec(execOptions?: ExecOptions): Promise<void>;
|
|
33
39
|
}
|
|
34
40
|
export declare class Exec extends FluentBase implements Executable {
|
|
35
|
-
exec(
|
|
41
|
+
exec(execOptions?: ExecOptions): Promise<void>;
|
|
36
42
|
}
|
|
37
43
|
export declare class FluentFilters extends FluentBase {
|
|
38
44
|
/**
|
|
@@ -808,7 +814,7 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
|
|
|
808
814
|
* @return {FluentFilters}
|
|
809
815
|
*/
|
|
810
816
|
contains(): FluentFilters;
|
|
811
|
-
exec(
|
|
817
|
+
exec(execOptions?: ExecOptions): Promise<void>;
|
|
812
818
|
}
|
|
813
819
|
export declare class FluentFiltersCondition extends FluentBase {
|
|
814
820
|
/**
|
|
@@ -1634,6 +1640,7 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
|
|
|
1634
1640
|
notExists(): ExecCondition;
|
|
1635
1641
|
}
|
|
1636
1642
|
declare class ExecCondition extends Exec {
|
|
1643
|
+
exec(execOptions?: ExecOptions): Promise<void>;
|
|
1637
1644
|
}
|
|
1638
1645
|
export declare abstract class FluentCommand extends FluentBase {
|
|
1639
1646
|
constructor();
|
|
@@ -2082,7 +2089,7 @@ export declare abstract class FluentCommand extends FluentBase {
|
|
|
2082
2089
|
* @return {Exec}
|
|
2083
2090
|
*/
|
|
2084
2091
|
pressAndroidKey(key: ANDROID_KEY): Exec;
|
|
2085
|
-
abstract fluentCommandExecutor(instruction: string, modelComposition: ModelCompositionBranch[], context: CommandExecutorContext): Promise<void>;
|
|
2092
|
+
abstract fluentCommandExecutor(instruction: string, modelComposition: ModelCompositionBranch[], context: CommandExecutorContext, skipCache: boolean, retryStrategy?: RetryStrategy): Promise<void>;
|
|
2086
2093
|
}
|
|
2087
2094
|
export interface ExecutableGetter {
|
|
2088
2095
|
exec(): Promise<DetectedElement[]>;
|