askui 0.30.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/cjs/core/cache/cache-config.d.ts +5 -0
  2. package/dist/cjs/core/cache/cache-config.js +2 -0
  3. package/dist/cjs/core/cache/cache-entry-reference.d.ts +7 -0
  4. package/dist/cjs/core/cache/cache-entry-reference.js +20 -0
  5. package/dist/cjs/core/cache/cache-entry.d.ts +11 -0
  6. package/dist/cjs/core/cache/cache-entry.js +47 -0
  7. package/dist/cjs/core/cache/cache-interface.d.ts +11 -0
  8. package/dist/cjs/core/cache/cache-interface.js +2 -0
  9. package/dist/cjs/core/cache/cache-manager.d.ts +24 -0
  10. package/dist/cjs/core/cache/cache-manager.js +145 -0
  11. package/dist/cjs/core/cache/cahe-file.d.ts +19 -0
  12. package/dist/cjs/core/cache/cahe-file.js +128 -0
  13. package/dist/cjs/core/cache/dummy-cache-manager.d.ts +12 -0
  14. package/dist/cjs/core/cache/dummy-cache-manager.js +27 -0
  15. package/dist/cjs/core/cache/image-reference.d.ts +10 -0
  16. package/dist/cjs/core/cache/image-reference.js +40 -0
  17. package/dist/cjs/core/cache/index.d.ts +7 -0
  18. package/dist/cjs/core/cache/index.js +13 -0
  19. package/dist/cjs/core/model/annotation-result/boundary-box.js +1 -1
  20. package/dist/cjs/core/model/custom-element.d.ts +1 -0
  21. package/dist/cjs/core/model/custom-element.js +3 -0
  22. package/dist/cjs/core/ui-control-commands/action.d.ts +1 -0
  23. package/dist/cjs/core/ui-control-commands/action.js +10 -2
  24. package/dist/cjs/core/ui-control-commands/control-command.d.ts +1 -0
  25. package/dist/cjs/core/ui-control-commands/control-command.js +7 -0
  26. package/dist/cjs/execution/dsl.d.ts +12 -5
  27. package/dist/cjs/execution/dsl.js +30 -15
  28. package/dist/cjs/execution/execution-runtime.d.ts +1 -1
  29. package/dist/cjs/execution/execution-runtime.js +21 -17
  30. package/dist/cjs/execution/inference-client.d.ts +5 -3
  31. package/dist/cjs/execution/inference-client.js +22 -3
  32. package/dist/cjs/execution/ui-control-client-dependency-builder.js +6 -1
  33. package/dist/cjs/execution/ui-control-client.d.ts +2 -1
  34. package/dist/cjs/execution/ui-control-client.js +28 -11
  35. package/dist/cjs/execution/ui-controller-client-interface.d.ts +2 -0
  36. package/dist/cjs/utils/base_64_image/base-64-image.d.ts +2 -0
  37. package/dist/cjs/utils/base_64_image/base-64-image.js +27 -4
  38. package/dist/esm/core/cache/cache-config.d.ts +5 -0
  39. package/dist/esm/core/cache/cache-config.js +1 -0
  40. package/dist/esm/core/cache/cache-entry-reference.d.ts +7 -0
  41. package/dist/esm/core/cache/cache-entry-reference.js +16 -0
  42. package/dist/esm/core/cache/cache-entry.d.ts +11 -0
  43. package/dist/esm/core/cache/cache-entry.js +43 -0
  44. package/dist/esm/core/cache/cache-interface.d.ts +11 -0
  45. package/dist/esm/core/cache/cache-interface.js +1 -0
  46. package/dist/esm/core/cache/cache-manager.d.ts +24 -0
  47. package/dist/esm/core/cache/cache-manager.js +141 -0
  48. package/dist/esm/core/cache/cahe-file.d.ts +19 -0
  49. package/dist/esm/core/cache/cahe-file.js +121 -0
  50. package/dist/esm/core/cache/dummy-cache-manager.d.ts +12 -0
  51. package/dist/esm/core/cache/dummy-cache-manager.js +23 -0
  52. package/dist/esm/core/cache/image-reference.d.ts +10 -0
  53. package/dist/esm/core/cache/image-reference.js +36 -0
  54. package/dist/esm/core/cache/index.d.ts +7 -0
  55. package/dist/esm/core/cache/index.js +5 -0
  56. package/dist/esm/core/model/annotation-result/boundary-box.js +1 -1
  57. package/dist/esm/core/model/custom-element.d.ts +1 -0
  58. package/dist/esm/core/model/custom-element.js +3 -0
  59. package/dist/esm/core/ui-control-commands/action.d.ts +1 -0
  60. package/dist/esm/core/ui-control-commands/action.js +10 -2
  61. package/dist/esm/core/ui-control-commands/control-command.d.ts +1 -0
  62. package/dist/esm/core/ui-control-commands/control-command.js +7 -0
  63. package/dist/esm/execution/dsl.d.ts +12 -5
  64. package/dist/esm/execution/dsl.js +30 -15
  65. package/dist/esm/execution/execution-runtime.d.ts +1 -1
  66. package/dist/esm/execution/execution-runtime.js +21 -17
  67. package/dist/esm/execution/inference-client.d.ts +5 -3
  68. package/dist/esm/execution/inference-client.js +22 -3
  69. package/dist/esm/execution/ui-control-client-dependency-builder.js +6 -1
  70. package/dist/esm/execution/ui-control-client.d.ts +2 -1
  71. package/dist/esm/execution/ui-control-client.js +28 -11
  72. package/dist/esm/execution/ui-controller-client-interface.d.ts +2 -0
  73. package/dist/esm/utils/base_64_image/base-64-image.d.ts +2 -0
  74. package/dist/esm/utils/base_64_image/base-64-image.js +27 -4
  75. package/package.json +1 -1
@@ -0,0 +1,43 @@
1
+ import { ControlCommand } from '../ui-control-commands';
2
+ import { CacheEntryReference } from './cache-entry-reference';
3
+ import { logger } from '../../lib';
4
+ export class CacheEntry {
5
+ constructor(alwaysValid, controlCommand, reference, createdAt = new Date()) {
6
+ this.alwaysValid = alwaysValid;
7
+ this.controlCommand = controlCommand;
8
+ this.reference = reference;
9
+ this.createdAt = createdAt;
10
+ }
11
+ static fromJson(json) {
12
+ try {
13
+ if (json === undefined) {
14
+ throw new Error('Cache entry is undefined');
15
+ }
16
+ if (json.controlCommand === undefined) {
17
+ throw new Error('the key "controlCommand" is required');
18
+ }
19
+ if (json.createdAt === undefined) {
20
+ throw new Error('the key "createdAt" is required');
21
+ }
22
+ if (json.alwaysValid === undefined) {
23
+ throw new Error('the key "alwaysValid" is required');
24
+ }
25
+ return new CacheEntry(json.alwaysValid, ControlCommand.fromJson(json.controlCommand), json.reference ? CacheEntryReference.fromJson(json.reference) : undefined, new Date(json.createdAt));
26
+ }
27
+ catch (error) {
28
+ logger.error(`Error deserializing cache entry: ${error}`);
29
+ return undefined;
30
+ }
31
+ }
32
+ toJson() {
33
+ const jsonObject = {
34
+ alwaysValid: this.alwaysValid,
35
+ controlCommand: this.controlCommand.toJson(),
36
+ createdAt: this.createdAt,
37
+ };
38
+ if (this.reference !== undefined) {
39
+ jsonObject['reference'] = this.reference.toJson();
40
+ }
41
+ return jsonObject;
42
+ }
43
+ }
@@ -0,0 +1,11 @@
1
+ import { CustomElement } from '../model/custom-element';
2
+ import { ControlCommand } from '../ui-control-commands';
3
+ import { CacheEntry } from './cache-entry';
4
+ export interface CacheInterface {
5
+ loadFromFile(): void;
6
+ saveToFile(): void;
7
+ add(key: string, value: CacheEntry): void;
8
+ addCacheEntryFromControlCommand(instruction: string, controlCommand: ControlCommand, customElements: CustomElement[], image?: string): Promise<void>;
9
+ isImageRequired(instruction: string, customElements: CustomElement[]): boolean | undefined;
10
+ getCachedControlCommand(instruction: string, customElements: CustomElement[], image?: string): Promise<ControlCommand | undefined>;
11
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,24 @@
1
+ import { ControlCommand } from '../ui-control-commands';
2
+ import { CacheInterface } from './cache-interface';
3
+ import { CacheEntry } from './cache-entry';
4
+ import { CacheConfig, ValidationType } from './cache-config';
5
+ import { CustomElement } from '../model/custom-element';
6
+ export declare class CacheManager implements CacheInterface {
7
+ private relevantData;
8
+ validationType: ValidationType;
9
+ private cacheFile;
10
+ private readonly referenceWidth;
11
+ private readonly referenceHeight;
12
+ private readonly defaultValidationType;
13
+ constructor(config: CacheConfig);
14
+ loadFromFile(): void;
15
+ saveToFile(): void;
16
+ add(key: string, value: CacheEntry): void;
17
+ addCacheEntryFromControlCommand(instruction: string, controlCommand: ControlCommand, customElements: CustomElement[], image?: string): Promise<void>;
18
+ isImageRequired(instruction: string, customElements: CustomElement[]): boolean | undefined;
19
+ getCachedControlCommand(instruction: string, customElements: CustomElement[], image?: string): Promise<ControlCommand | undefined>;
20
+ private encodeKey;
21
+ private getCacheEntriesByKey;
22
+ private getValidCacheEntry;
23
+ private validateAccordingToPixelPerfect;
24
+ }
@@ -0,0 +1,141 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { ControlCommandCode } from '../ui-control-commands';
11
+ import { InputEvent } from '../ui-control-commands/input-event';
12
+ import { Base64Image } from '../../utils/base_64_image/base-64-image';
13
+ import { CacheEntry } from './cache-entry';
14
+ import { ImageReference } from './image-reference';
15
+ import { CacheEntryReference } from './cache-entry-reference';
16
+ import { CacheFile } from './cahe-file';
17
+ export class CacheManager {
18
+ constructor(config) {
19
+ var _a;
20
+ this.relevantData = {};
21
+ this.referenceWidth = 32;
22
+ this.referenceHeight = 16;
23
+ this.defaultValidationType = 'PixelPerfect';
24
+ this.validationType = (_a = config.validationType) !== null && _a !== void 0 ? _a : this.defaultValidationType;
25
+ this.cacheFile = new CacheFile(config.cacheFilePath);
26
+ }
27
+ loadFromFile() {
28
+ this.cacheFile.loadFromFile();
29
+ this.relevantData = this.cacheFile.getDataForValidationType(this.validationType);
30
+ }
31
+ saveToFile() {
32
+ this.cacheFile.saveToFile(this.validationType, this.relevantData);
33
+ }
34
+ add(key, value) {
35
+ if (this.relevantData[key] === undefined) {
36
+ this.relevantData[key] = [];
37
+ }
38
+ this.relevantData[key].push(value);
39
+ }
40
+ addCacheEntryFromControlCommand(instruction, controlCommand, customElements, image) {
41
+ return __awaiter(this, void 0, void 0, function* () {
42
+ let imageReference;
43
+ if (image !== undefined) {
44
+ const moveAction = controlCommand.actions.find((action) => action.inputEvent === InputEvent.MOUSE_MOVE);
45
+ const x = moveAction === null || moveAction === void 0 ? void 0 : moveAction.position.x;
46
+ const y = moveAction === null || moveAction === void 0 ? void 0 : moveAction.position.y;
47
+ if (x !== undefined && y !== undefined) {
48
+ const xTopLeft = Math.max(0, (x - this.referenceWidth / 2));
49
+ const yTopLeft = Math.max(0, (y - this.referenceHeight / 2));
50
+ const base64Image = yield Base64Image.fromString(image);
51
+ const croppedImage = yield base64Image.cropRegion(xTopLeft, yTopLeft, this.referenceWidth, this.referenceHeight);
52
+ imageReference = new ImageReference(croppedImage.toString(true), this.referenceWidth, this.referenceHeight, xTopLeft, yTopLeft);
53
+ }
54
+ }
55
+ const cacheEntry = new CacheEntry(image === undefined, controlCommand, new CacheEntryReference(imageReference));
56
+ const cacheKey = this.encodeKey(instruction, customElements);
57
+ this.add(cacheKey, cacheEntry);
58
+ });
59
+ }
60
+ isImageRequired(instruction, customElements) {
61
+ const cacheKey = this.encodeKey(instruction, customElements);
62
+ const cacheEntries = this.getCacheEntriesByKey(cacheKey);
63
+ if (cacheEntries.length === 0) {
64
+ return undefined;
65
+ }
66
+ return cacheEntries.find((entry) => entry.alwaysValid === false) !== undefined;
67
+ }
68
+ getCachedControlCommand(instruction, customElements, image) {
69
+ return __awaiter(this, void 0, void 0, function* () {
70
+ const cacheKey = this.encodeKey(instruction, customElements);
71
+ const cacheEntry = yield this.getValidCacheEntry(cacheKey, image);
72
+ if (cacheEntry === undefined || cacheEntry.controlCommand.code !== ControlCommandCode.OK) {
73
+ return undefined;
74
+ }
75
+ return cacheEntry.controlCommand;
76
+ });
77
+ }
78
+ // eslint-disable-next-line class-methods-use-this
79
+ encodeKey(instruction, customElements) {
80
+ let key = instruction;
81
+ if (customElements.length > 0) {
82
+ key += ` with CustomElements:${customElements.map((element, index) => `${index}:${element.asString()}`).join(',')}`;
83
+ }
84
+ return key;
85
+ }
86
+ getCacheEntriesByKey(key) {
87
+ if (this.relevantData[key] === undefined) {
88
+ return [];
89
+ }
90
+ const entries = this.relevantData[key];
91
+ if (entries === undefined) {
92
+ return [];
93
+ }
94
+ return entries;
95
+ }
96
+ getValidCacheEntry(key, screenshot) {
97
+ return __awaiter(this, void 0, void 0, function* () {
98
+ const cacheEntries = this.getCacheEntriesByKey(key);
99
+ if (cacheEntries.length === 0) {
100
+ return undefined;
101
+ }
102
+ if (this.validationType === 'PixelPerfect') {
103
+ /* eslint-disable no-restricted-syntax, no-await-in-loop */
104
+ for (const cacheEntry of cacheEntries) {
105
+ if (cacheEntry.alwaysValid) {
106
+ return cacheEntry;
107
+ }
108
+ const isValid = yield this.validateAccordingToPixelPerfect(cacheEntry, screenshot);
109
+ if (isValid) {
110
+ /* eslint-enable no-restricted-syntax, no-await-in-loop */
111
+ return cacheEntry;
112
+ }
113
+ }
114
+ /* eslint-enable no-restricted-syntax, no-await-in-loop */
115
+ }
116
+ return undefined;
117
+ });
118
+ }
119
+ // eslint-disable-next-line class-methods-use-this
120
+ validateAccordingToPixelPerfect(cacheEntry, screenshot) {
121
+ return __awaiter(this, void 0, void 0, function* () {
122
+ var _a;
123
+ if (screenshot === undefined) {
124
+ return false;
125
+ }
126
+ if (((_a = cacheEntry.reference) === null || _a === void 0 ? void 0 : _a.image) === undefined) {
127
+ return false;
128
+ }
129
+ try {
130
+ const referenceImage = yield Base64Image.fromString(screenshot);
131
+ const croppedScreenshot = yield referenceImage.cropRegion(cacheEntry.reference.image.xTopLeft, cacheEntry.reference.image.yTopLeft, cacheEntry.reference.image.width, cacheEntry.reference.image.height);
132
+ const croppedReferenceImageString = croppedScreenshot.toString(true);
133
+ const isValid = croppedReferenceImageString === cacheEntry.reference.image.base64Image;
134
+ return isValid;
135
+ }
136
+ catch (error) {
137
+ return false;
138
+ }
139
+ });
140
+ }
141
+ }
@@ -0,0 +1,19 @@
1
+ import { CacheEntry } from './cache-entry';
2
+ import { ValidationType } from './cache-config';
3
+ export type CacheDataByInstruction = Record<string, CacheEntry[]>;
4
+ export type CacheDataByValidationType = Record<string, CacheDataByInstruction>;
5
+ export declare class CacheFile {
6
+ private createAt;
7
+ private version;
8
+ private type;
9
+ private data;
10
+ private filePath;
11
+ constructor(filePath?: string);
12
+ loadFromFileVersion1(json: CacheFile): void;
13
+ loadFromFile(): void;
14
+ getDataForValidationType(validationType: ValidationType): CacheDataByInstruction;
15
+ getData(): CacheDataByValidationType;
16
+ saveToFile(validationType: ValidationType, relevantData: CacheDataByInstruction): void;
17
+ private setRelevantData;
18
+ private asJsonObject;
19
+ }
@@ -0,0 +1,121 @@
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import { logger } from '../../lib';
4
+ import { CacheEntry } from './cache-entry';
5
+ export class CacheFile {
6
+ constructor(filePath) {
7
+ this.createAt = new Date();
8
+ this.version = 1;
9
+ this.type = 'AskUI-Cache';
10
+ this.data = {};
11
+ this.filePath = filePath;
12
+ }
13
+ loadFromFileVersion1(json) {
14
+ if (json.version !== 1) {
15
+ throw new Error(`Unsupported key 'version' in the cache file: '${json.version}'. Only version 1 is supported.`);
16
+ }
17
+ if (json.data === undefined) {
18
+ throw new Error("Key 'data' is required in the cache file.");
19
+ }
20
+ if (json.createAt === undefined) {
21
+ throw new Error("Key 'createAt' is required in the cache file.");
22
+ }
23
+ if (json.version === undefined) {
24
+ throw new Error("Key 'version' is required in the cache file.");
25
+ }
26
+ if (json.type !== 'AskUI-Cache') {
27
+ throw new Error(`Unsupported key 'type' in the cache file: '${json.type}'. Only 'AskUI-Cache' is supported.`);
28
+ }
29
+ this.createAt = new Date(json.createAt);
30
+ this.version = json.version;
31
+ this.type = json.type;
32
+ this.data = Object.keys(json.data).reduce((acc, validationType) => {
33
+ var _a, _b;
34
+ const nextAcc = Object.assign({}, acc);
35
+ const validationEntries = (_b = (_a = json.data) === null || _a === void 0 ? void 0 : _a[validationType]) !== null && _b !== void 0 ? _b : {};
36
+ nextAcc[validationType] = Object.keys(validationEntries).reduce((innerAcc, instruction) => {
37
+ var _a;
38
+ const nextInnerAcc = Object.assign({}, innerAcc);
39
+ const rawEntries = (_a = validationEntries[instruction]) !== null && _a !== void 0 ? _a : [];
40
+ nextInnerAcc[instruction] = rawEntries
41
+ .map((entry) => CacheEntry.fromJson(entry))
42
+ .filter((entry) => entry !== undefined);
43
+ return nextInnerAcc;
44
+ }, {});
45
+ return nextAcc;
46
+ }, {});
47
+ }
48
+ loadFromFile() {
49
+ if (this.filePath === undefined) {
50
+ logger.debug('The cache file path is not set. Skipping the cache load.');
51
+ return;
52
+ }
53
+ logger.debug(`Loading the cache from file '${this.filePath}'.`);
54
+ if (!fs.existsSync(this.filePath)) {
55
+ logger.warn(`The cache file does not exist: '${this.filePath}'. Skipping the cache load.`);
56
+ return;
57
+ }
58
+ const data = fs.readFileSync(this.filePath, 'utf8');
59
+ const json = JSON.parse(data);
60
+ if (json.version !== 1) {
61
+ throw new Error(`Unsupported key 'version' in the cache file: '${json.version}'. Only version 1 is supported.`);
62
+ }
63
+ this.loadFromFileVersion1(json);
64
+ }
65
+ getDataForValidationType(validationType) {
66
+ var _a;
67
+ return (_a = this.data[validationType]) !== null && _a !== void 0 ? _a : {};
68
+ }
69
+ getData() {
70
+ return this.data;
71
+ }
72
+ saveToFile(validationType, relevantData) {
73
+ if (this.filePath === undefined) {
74
+ logger.debug('The cache file path is not set. Skipping the cache save.');
75
+ return;
76
+ }
77
+ try {
78
+ logger.debug(`Saving the cache to file '${this.filePath}'.`);
79
+ this.setRelevantData(validationType, relevantData);
80
+ const jsonObject = this.asJsonObject();
81
+ const jsonString = JSON.stringify(jsonObject, null, 2);
82
+ // create the directory if it doesn't exist
83
+ if (!fs.existsSync(path.dirname(this.filePath))) {
84
+ fs.mkdirSync(path.dirname(this.filePath), { recursive: true });
85
+ }
86
+ fs.writeFileSync(this.filePath, jsonString, 'utf8');
87
+ logger.info(`The cache was saved successfully to '${this.filePath}'.`);
88
+ }
89
+ catch (error) {
90
+ logger.error(`An error occurred while saving the cache to file '${this.filePath}': '${error}'`);
91
+ throw error;
92
+ }
93
+ }
94
+ setRelevantData(validationType, relevantData) {
95
+ this.data[validationType] = relevantData;
96
+ }
97
+ asJsonObject() {
98
+ return {
99
+ createAt: this.createAt.toISOString(),
100
+ data: Object.keys(this.data).reduce((acc, validationType) => {
101
+ const validationData = this.data[validationType];
102
+ if (validationData === undefined) {
103
+ return acc;
104
+ }
105
+ const nextAcc = Object.assign({}, acc);
106
+ nextAcc[validationType] = Object.keys(validationData).reduce((innerAcc, instruction) => {
107
+ const entries = validationData[instruction];
108
+ if (entries !== undefined) {
109
+ const nextInnerAcc = Object.assign({}, innerAcc);
110
+ nextInnerAcc[instruction] = entries;
111
+ return nextInnerAcc;
112
+ }
113
+ return innerAcc;
114
+ }, {});
115
+ return nextAcc;
116
+ }, {}),
117
+ type: this.type,
118
+ version: this.version,
119
+ };
120
+ }
121
+ }
@@ -0,0 +1,12 @@
1
+ import { CacheInterface } from './cache-interface';
2
+ import { CacheEntry } from './cache-entry';
3
+ import { ControlCommand } from '../ui-control-commands';
4
+ import { CustomElement } from '../model/custom-element';
5
+ export declare class DummyCacheManager implements CacheInterface {
6
+ add(_key: string, _value: CacheEntry): void;
7
+ addCacheEntryFromControlCommand(_instruction: string, _controlCommand: ControlCommand, _customElements: CustomElement[], _image?: string): Promise<void>;
8
+ isImageRequired(_instruction: string, _customElements: CustomElement[]): boolean | undefined;
9
+ getCachedControlCommand(_instruction: string, _customElements: CustomElement[], _image?: string): Promise<ControlCommand | undefined>;
10
+ loadFromFile(): void;
11
+ saveToFile(): void;
12
+ }
@@ -0,0 +1,23 @@
1
+ export class DummyCacheManager {
2
+ // eslint-disable-next-line class-methods-use-this
3
+ add(_key, _value) {
4
+ }
5
+ // eslint-disable-next-line class-methods-use-this
6
+ addCacheEntryFromControlCommand(_instruction, _controlCommand, _customElements, _image) {
7
+ return Promise.resolve();
8
+ }
9
+ // eslint-disable-next-line class-methods-use-this
10
+ isImageRequired(_instruction, _customElements) {
11
+ return undefined;
12
+ }
13
+ // eslint-disable-next-line class-methods-use-this
14
+ getCachedControlCommand(_instruction, _customElements, _image) {
15
+ return Promise.resolve(undefined);
16
+ }
17
+ // eslint-disable-next-line class-methods-use-this
18
+ loadFromFile() {
19
+ }
20
+ // eslint-disable-next-line class-methods-use-this
21
+ saveToFile() {
22
+ }
23
+ }
@@ -0,0 +1,10 @@
1
+ export declare class ImageReference {
2
+ base64Image: string;
3
+ width: number;
4
+ height: number;
5
+ xTopLeft: number;
6
+ yTopLeft: number;
7
+ constructor(base64Image: string, width: number, height: number, xTopLeft: number, yTopLeft: number);
8
+ static fromJson(json: ImageReference): ImageReference;
9
+ toJson(): Record<string, unknown>;
10
+ }
@@ -0,0 +1,36 @@
1
+ export class ImageReference {
2
+ constructor(base64Image, width, height, xTopLeft, yTopLeft) {
3
+ this.base64Image = base64Image;
4
+ this.width = width;
5
+ this.height = height;
6
+ this.xTopLeft = xTopLeft;
7
+ this.yTopLeft = yTopLeft;
8
+ }
9
+ static fromJson(json) {
10
+ if (json.base64Image === undefined) {
11
+ throw new Error('the key "base64Image" is required');
12
+ }
13
+ if (json.width === undefined) {
14
+ throw new Error('the key "width" is required');
15
+ }
16
+ if (json.height === undefined) {
17
+ throw new Error('the key "height" is required');
18
+ }
19
+ if (json.xTopLeft === undefined) {
20
+ throw new Error('the key "xTopLeft" is required');
21
+ }
22
+ if (json.yTopLeft === undefined) {
23
+ throw new Error('the key "yTopLeft" is required');
24
+ }
25
+ return new ImageReference(json.base64Image, json.width, json.height, json.xTopLeft, json.yTopLeft);
26
+ }
27
+ toJson() {
28
+ return {
29
+ base64Image: this.base64Image,
30
+ height: this.height,
31
+ width: this.width,
32
+ xTopLeft: this.xTopLeft,
33
+ yTopLeft: this.yTopLeft,
34
+ };
35
+ }
36
+ }
@@ -0,0 +1,7 @@
1
+ export { ImageReference } from './image-reference';
2
+ export { CacheEntryReference } from './cache-entry-reference';
3
+ export { CacheEntry } from './cache-entry';
4
+ export { CacheConfig, ValidationType } from './cache-config';
5
+ export { CacheInterface } from './cache-interface';
6
+ export { DummyCacheManager } from './dummy-cache-manager';
7
+ export { CacheManager } from './cache-manager';
@@ -0,0 +1,5 @@
1
+ export { ImageReference } from './image-reference';
2
+ export { CacheEntryReference } from './cache-entry-reference';
3
+ export { CacheEntry } from './cache-entry';
4
+ export { DummyCacheManager } from './dummy-cache-manager';
5
+ export { CacheManager } from './cache-manager';
@@ -17,7 +17,7 @@ export class BoundingBox {
17
17
  this.ymax = ymax;
18
18
  }
19
19
  static fromJson(boundinBox, resizeRatio = 1) {
20
- return new BoundingBox(boundinBox.xmin * resizeRatio, boundinBox.ymin * resizeRatio, boundinBox.xmax * resizeRatio, boundinBox.ymax * resizeRatio);
20
+ return new BoundingBox(Math.round(boundinBox.xmin * resizeRatio), Math.round(boundinBox.ymin * resizeRatio), Math.round(boundinBox.xmax * resizeRatio), Math.round(boundinBox.ymax * resizeRatio));
21
21
  }
22
22
  /**
23
23
  *
@@ -19,4 +19,5 @@ export declare class CustomElement implements CustomElementJson {
19
19
  static fromJsonWithImagePathOrImage(ceJson: CustomElementJson): Promise<CustomElement>;
20
20
  static fromJson(ceJson: CustomElementJson): CustomElement;
21
21
  validate(): void;
22
+ asString(): string;
22
23
  }
@@ -43,6 +43,9 @@ export class CustomElement {
43
43
  throw new ValidationError(e.errors.join(', '));
44
44
  }
45
45
  }
46
+ asString() {
47
+ return `name:${this.name}-customImage:${this.customImage}-threshold:${this.threshold}-stopThreshold:${this.stopThreshold}-rotationDegreePerStep:${this.rotationDegreePerStep}-imageCompareFormat:${this.imageCompareFormat}-mask:${this.mask}`;
48
+ }
46
49
  }
47
50
  CustomElement.schema = object({
48
51
  mask: array().optional().min(3, 'mask must contain at least 3 points'),
@@ -15,4 +15,5 @@ export declare class Action {
15
15
  y: number;
16
16
  }, text: string, parameters?: ActionParameters);
17
17
  static fromJson(action: Action, resizeRatio?: number): Action;
18
+ toJson(): object;
18
19
  }
@@ -8,8 +8,16 @@ export class Action {
8
8
  }
9
9
  static fromJson(action, resizeRatio = 1) {
10
10
  return new Action(InputEvent[action.inputEvent], {
11
- x: action.position.x * resizeRatio,
12
- y: action.position.y * resizeRatio,
11
+ x: Math.round(action.position.x * resizeRatio),
12
+ y: Math.round(action.position.y * resizeRatio),
13
13
  }, action.text, action.parameters ? action.parameters : {});
14
14
  }
15
+ toJson() {
16
+ return {
17
+ inputEvent: this.inputEvent,
18
+ parameters: this.parameters,
19
+ position: this.position,
20
+ text: this.text,
21
+ };
22
+ }
15
23
  }
@@ -7,4 +7,5 @@ export declare class ControlCommand {
7
7
  constructor(code: ControlCommandCode, actions: Action[], tryToRepeat?: boolean);
8
8
  static fromJson(json: unknown, resizeRatio?: number): ControlCommand;
9
9
  setTextToBeTyped(text: string): void;
10
+ toJson(): object;
10
11
  }
@@ -15,4 +15,11 @@ export class ControlCommand {
15
15
  this.actions = this.actions.map((action) => ([InputEvent.TYPE, InputEvent.TYPE_TEXT].includes(action.inputEvent)
16
16
  ? new Action(action.inputEvent, action.position, text, action.parameters) : action));
17
17
  }
18
+ toJson() {
19
+ return {
20
+ actions: this.actions.map((action) => action.toJson()),
21
+ code: this.code,
22
+ tryToRepeat: this.tryToRepeat,
23
+ };
24
+ }
18
25
  }
@@ -1,6 +1,7 @@
1
1
  import { CustomElementJson } from '../core/model/custom-element-json';
2
2
  import { DetectedElement } from '../core/model/annotation-result/detected-element';
3
3
  import { ModelCompositionBranch } from './model-composition-branch';
4
+ import { RetryStrategy } from './retry-strategies/retry-strategy';
4
5
  export declare enum Separators {
5
6
  STRING = "<|string|>"
6
7
  }
@@ -17,22 +18,27 @@ export interface CommandExecutorContext {
17
18
  customElementsJson: CustomElementJson[];
18
19
  aiElementNames: string[];
19
20
  }
21
+ export interface ExecOptions {
22
+ modelComposition?: ModelCompositionBranch[];
23
+ skipCache?: boolean;
24
+ retryStrategy?: RetryStrategy;
25
+ }
20
26
  declare abstract class FluentBase {
21
27
  protected prev?: FluentBase | undefined;
22
28
  constructor(prev?: FluentBase | undefined);
23
29
  protected _textStr: string;
24
30
  protected _params: Map<string, unknown>;
25
31
  protected static addParams(paramsList: Map<string, unknown[]>, params: Map<string, unknown>): Map<string, unknown[]>;
26
- protected fluentCommandStringBuilder(modelComposition: ModelCompositionBranch[], currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<void>;
32
+ protected fluentCommandStringBuilder(modelComposition?: ModelCompositionBranch[], skipCache?: boolean, retryStrategy?: RetryStrategy, currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<void>;
27
33
  protected getterStringBuilder(currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<DetectedElement[]>;
28
34
  protected get textStr(): string;
29
35
  protected get params(): Map<string, unknown>;
30
36
  }
31
37
  export interface Executable {
32
- exec(): Promise<void>;
38
+ exec(execOptions?: ExecOptions): Promise<void>;
33
39
  }
34
40
  export declare class Exec extends FluentBase implements Executable {
35
- exec(modelComposition?: ModelCompositionBranch[]): Promise<void>;
41
+ exec(execOptions?: ExecOptions): Promise<void>;
36
42
  }
37
43
  export declare class FluentFilters extends FluentBase {
38
44
  /**
@@ -808,7 +814,7 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
808
814
  * @return {FluentFilters}
809
815
  */
810
816
  contains(): FluentFilters;
811
- exec(modelComposition?: ModelCompositionBranch[]): Promise<void>;
817
+ exec(execOptions?: ExecOptions): Promise<void>;
812
818
  }
813
819
  export declare class FluentFiltersCondition extends FluentBase {
814
820
  /**
@@ -1634,6 +1640,7 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
1634
1640
  notExists(): ExecCondition;
1635
1641
  }
1636
1642
  declare class ExecCondition extends Exec {
1643
+ exec(execOptions?: ExecOptions): Promise<void>;
1637
1644
  }
1638
1645
  export declare abstract class FluentCommand extends FluentBase {
1639
1646
  constructor();
@@ -2082,7 +2089,7 @@ export declare abstract class FluentCommand extends FluentBase {
2082
2089
  * @return {Exec}
2083
2090
  */
2084
2091
  pressAndroidKey(key: ANDROID_KEY): Exec;
2085
- abstract fluentCommandExecutor(instruction: string, modelComposition: ModelCompositionBranch[], context: CommandExecutorContext): Promise<void>;
2092
+ abstract fluentCommandExecutor(instruction: string, modelComposition: ModelCompositionBranch[], context: CommandExecutorContext, skipCache: boolean, retryStrategy?: RetryStrategy): Promise<void>;
2086
2093
  }
2087
2094
  export interface ExecutableGetter {
2088
2095
  exec(): Promise<DetectedElement[]>;